# # Collective Knowledge (package) # # See CK LICENSE.txt for licensing details # See CK COPYRIGHT.txt for copyright details # # Developer: Grigori Fursin, Grigori.Fursin@cTuning.org, http://fursin.net # cfg={} # Will be updated by CK (meta description of this module) work={} # Will be updated by CK (temporal data) ck=None # Will be updated by CK (initialized CK kernel) # Local settings fix_env_for_rebuild={"PACKAGE_GIT": "NO", "PACKAGE_WGET": "NO", "PACKAGE_PATCH": "NO", "PACKAGE_PATCH": "NO", "PACKAGE_UNGZIP": "NO", "PACKAGE_UNTAR": "NO", "PACKAGE_UNBZIP": "NO", "PACKAGE_SKIP_CLEAN_INSTALL": "YES", "PACKAGE_SKIP_CLEAN_OBJ": "YES", "PACKAGE_SKIP_CLEAN_SRC_DIR": "YES"} ############################################################################## # Initialize module def init(i): """ Input: {} Output: { return - return code = 0, if successful > 0, if error (error) - error text if return > 0 } """ return {'return':0} ############################################################################## # install package def install(i): """ Input: { (target) - if specified, use info from 'machine' module or (host_os) - host OS (detect, if omitted) (target_os) - target OS (detect, if omitted) (target_device_id) - target device ID (detect, if omitted) (data_uoa) or (uoa) - package UOA entry or (tags) - tags to search package if data_uoa=='' before searching in current path (or_tags) - add entries which has groups of tags separated by ; (no_tags) - exclude entris with these tags separated by comma (env_data_uoa) - use this data UOA to record (new) env (env_repo_uoa) - use this repo to record new env (install_path) - full path with soft is installed (path) - path with soft is installed (path from the package will be appended) (skip_process) - if 'yes', skip archive processing (skip_setup) - if 'yes', skip environment setup (deps) - pre-set some deps, for example for compiler (reuse_deps) - if 'yes' reuse all deps if found in cache by tags (deps_cache) - list with resolved deps (deps.{KEY}) - set deps[KEY]["uoa']=value (user-friendly interface via CMD to set any given dependency) (preset_deps) - dict with {"KEY":"UOA"} to preset dependencies (dep_add_tags.{KEY}) - extra tags added to specific subdictionary of deps{} for this particular resolution session (param) - string converted into CK_PARAM and passed to processing script (params) - dict, keys are converted into = and passed to processing script (env_prefix) - alternative env_prefix (overrides the one in soft entry) (env) - add environment vars (env.{KEY}) - set env[KEY]=value (user-friendly interface via CMD) (Dkey) - update params[key], i.e. ck install package:... -DENV1=val1 -DENV2=val2 (similar to CMAKE) (extra_version) - add extra version, when registering software (for example, -trunk-20160421) (extra_tags) - add extra tags to separated customized packages (string separated by comma) (extra_path) - add extra path to the automatically prepared one (for example, -trunk-20160421) (record_script) - record tmp installation script with pre-set environment (to be able to call it to rebuild package without CK) (force_version) - force version (useful for automatic installation of packages with multiple supported versions) (install_to_env) - install this package and all dependencies to env instead of CK-TOOLS (to keep it clean)! (safe) - safe mode when searching packages first instead of detecting already installed soft (to have more deterministic build) (add_hint) - if 'yes', add hint that can skip package installation and detect soft instead (rebuild) - if 'yes', attempt to set env to avoid downloading package again, just rebuild (if supported) (reinstall) - if 'yes', also download package and then rebuild it ... (version_from) - check version starting from ... (list of numbers) (version_to) - check version up to ... (list of numbers) (ask) - if 'yes', ask more questions, otherwise select default actions (ask_version) - ask for the version of the package the user wants to install (debug) - if 'yes', open shell before installing but with all resolved deps } Output: { return - return code = 0, if successful > 0, if error (error) - error text if return > 0 env_data_uoa - if installed fine env_data_uid - if installed fine } """ import os import time import copy o=i.get('out','') oo='' if o=='con': oo=o ask=i.get('ask','') xtags=i.get('tags','') xor_tags=i.get('or_tags','') xno_tags=i.get('no_tags','') reuse_deps=i.get('reuse_deps','') deps_cache=i.get('deps_cache',[]) debug=i.get('debug','') # Check if package_channel is specifies and add tag pchannel=ck.cfg.get('package_channel','') if pchannel!='': if xtags!='': xtags+=',' xtags+='channel-'+pchannel start_time = time.time() # Check if target if i.get('target','')!='': r=ck.access({'action':'init', 'module_uoa':cfg['module_deps']['machine'], 'input':i}) if r['return']>0: return r device_cfg=i.get('device_cfg',{}) # Check host/target OS/CPU hos=i.get('host_os','') tos=i.get('target_os','') tdid=i.get('target_device_id','') r=ck.access({'action':'detect', 'module_uoa':cfg['module_deps']['platform.os'], 'host_os':hos, 'target_os':tos, 'device_cfg':device_cfg, 'target_device_id':tdid, 'skip_info_collection':'yes'}) if r['return']>0: return r hos=r['host_os_uid'] hosx=r['host_os_uoa'] hosd=r['host_os_dict'] tos=r['os_uid'] tosx=r['os_uoa'] tosd=r['os_dict'] host_add_path_string=r.get('host_add_path_string','') hosn=hosd.get('ck_name2','') osn=tosd.get('ck_name2','') # Check if base is different x1=hosd.get('base_uid','') x2=hosd.get('base_uoa','') if x1!='' and x2!='': hos=x1 hosx=x2 x1=tosd.get('base_uid','') x2=tosd.get('base_uoa','') if x1!='' and x2!='': tos=x1 tosx=x2 tbits=tosd.get('bits','') ck_os_name=hosd['ck_name'] tname2=tosd['ck_name2'] macos = tosd.get('macos') == 'yes' vfrom=i.get('version_from',[]) vto=i.get('version_to',[]) if type(vfrom)!=list: rx=ck.access({'action':'split_version', 'module_uoa':cfg['module_deps']['soft'], 'version':vfrom}) if rx['return']>0: return rx vfrom=rx['version_split'] if type(vto)!=list: rx=ck.access({'action':'split_version', 'module_uoa':cfg['module_deps']['soft'], 'version':vto}) if rx['return']>0: return rx vto=rx['version_split'] rem=hosd.get('rem','') eset=hosd.get('env_set','') svarb=hosd.get('env_var_start','') svare=hosd.get('env_var_stop','') scall=hosd.get('env_call','') sdirs=hosd.get('dir_sep','') sext=hosd.get('script_ext','') evs=hosd.get('env_var_separator','') eifs=hosd.get('env_quotes_if_space','') eifsc=hosd.get('env_quotes_if_space_in_call','') wb=tosd.get('windows_base','') iev=i.get('install_to_env','') if iev=='': iev=ck.cfg.get('install_to_env','') safe=i.get('safe','') rebuild=i.get('rebuild','') reinstall=i.get('reinstall','') force_version = i.get('force_version', '') # Check package description duoa=i.get('uoa','') if duoa=='': duoa=i.get('data_uoa','') requested_muoa=i.get('module_uoa','') duid='' package_repo_uoa='' d={} required_variations = [] if duoa=='' and requested_muoa=='': # Try to detect CID in current path rx=ck.detect_cid_in_current_path({}) if rx['return']==0: duoa=rx.get('data_uoa','') package_repo_uoa=rx.get('repo_uoa','') if xtags: # if tags are available, try searching both in tags and variations r=ck.access({'action': 'search_in_variations', 'data_uoa': duoa, 'module_uoa': 'misc', 'query_module_uoa': work['self_module_uid'], 'tags': xtags, 'add_info': 'yes', }) if r['return']>0: return r l=r['lst'] if len(l)>0: # Check that support host/target OS ll=[] for q in l: # Check if restricts dependency to a given host or target OS rx=ck.access({'action':'check_target', 'module_uoa':cfg['module_deps']['soft'], 'dict':q.get('meta',{}), 'host_os_uoa':hosx, 'host_os_dict':hosd, 'target_os_uoa':tosx, 'target_os_dict':tosd}) if rx['return']==0: specific_version = q.get('meta',{}).get('customize',{}).get('version','') supported_versions = q.get('meta',{}).get('customize',{}).get('supported_versions') if not specific_version and supported_versions and force_version=='': for s_version in supported_versions: q_clone = copy.deepcopy( q ) q_clone['meta']['customize']['version'] = s_version ll.append( q_clone ) else: ll.append(q) rx=ck.access({'action':'prune_search_list', 'module_uoa':cfg['module_deps']['env'], 'lst':ll, 'version_from':vfrom, 'version_to':vto, 'or_tags':xor_tags, 'no_tags':xno_tags}) if rx['return']>0: return rx ll=rx['lst'] # Select package if len(ll)>0: # Sort by name and version l=sorted(ll, key=lambda k: (k.get('meta',{}).get('sort', 0), internal_get_val(k.get('meta',{}).get('customize',{}).get('version_split',[]), 0, 0), internal_get_val(k.get('meta',{}).get('customize',{}).get('version_split',[]), 1, 0), internal_get_val(k.get('meta',{}).get('customize',{}).get('version_split',[]), 2, 0), internal_get_val(k.get('meta',{}).get('customize',{}).get('version_split',[]), 3, 0), internal_get_val(k.get('meta',{}).get('customize',{}).get('version_split',[]), 4, 0), k.get('info',{}).get('data_name',''), k['data_uoa']), reverse=True) selected_index=0 if len(l)>1: ver_options = [] ck.out('') ck.out('More than one package or version found:') ck.out('') display_to_idx={} for list_idx in range(len(l)): package_entry = l[list_idx] this_data_uid=package_entry['data_uid'] data_name_or_uoa = package_entry.get('info',{}).get('data_name', package_entry['data_uoa']) dmeta=package_entry.get('meta',{}) declared_version = dmeta.get('customize',{}).get('version','') version_if_defined = ' Version {} '.format(declared_version) if declared_version else '' declared_comment = dmeta.get('comment','') comment_in_braces = (declared_comment + ', ' if declared_comment else '') + this_data_uid required_variations = package_entry.get('required_variations') variations_to_display = ', Variations: {}'.format(','.join(required_variations)) if required_variations else '' display_line = '{}{} ({}){}'.format(data_name_or_uoa, version_if_defined, comment_in_braces, variations_to_display) display_to_idx[ display_line ] = list_idx ver_options.append( display_line ) skip_display_line = 'Skip CK package installation and attempt to detect installed soft' if i.get('add_hint','')=='yes': ver_options.append( skip_display_line ) select_adict = ck.access({'action': 'select_string', 'module_uoa': 'misc', 'options': ver_options, 'default': '0', 'no_skip_line': 'yes', 'question': 'Please select the package to install', }) if select_adict['return']>0: return select_adict if select_adict['selected_value']==skip_display_line: return {'return':16, 'error':'skipped package installation!'} selected_index = select_adict['selected_index'] ck.out('') package_entry = l[selected_index] duid=package_entry.get('data_uid','') duoa=duid duoax=package_entry.get('data_uoa','') package_repo_uoa=package_entry.get('repo_uoa','') d=package_entry['meta'] p=package_entry['path'] required_variations = package_entry.get('required_variations',[]) if o=='con': ck.out('') ck.out(' Package found: '+duoax+' ('+duid+')') ck.out('') if duoa=='': x='' if xor_tags!='': x='and with or_tags="'+xor_tags+'" ' if xno_tags!='': x='and with no_tags="'+xno_tags+'" ' return {'return':16, 'error':'package with tags "'+xtags+'" '+x+'for your environment was not found!'} elif duoa: # if tags were not available, try to load directly rx=ck.access({'action':'load', 'module_uoa':work['self_module_uid'], 'data_uoa':duoa}) if rx['return']>0: return rx d=rx['dict'] p=rx['path'] duoa=rx['data_uoa'] duid=rx['data_uid'] package_repo_uoa=rx.get('repo_uoa','') # Check if restricts dependency to a given host or target OS rx=ck.access({'action':'check_target', 'module_uoa':cfg['module_deps']['soft'], 'dict':d, 'host_os_uoa':hosx, 'host_os_dict':hosd, 'target_os_uoa':tosx, 'target_os_dict':tosd}) if rx['return']>0: return rx # Get main params tags=copy.deepcopy(d.get('tags',[])) extra_cli_tags = i.get('extra_tags','').strip() if extra_cli_tags!='': extra_cli_tags = extra_cli_tags.split(',') tags.extend(extra_cli_tags) else: extra_cli_tags = [] vari_and_cli_tags = set(extra_cli_tags) # to be topped up with variations, serialized and added to the install path udeps=d.get('deps',{}) depsx=i.get('deps',{}) if len(depsx)>0: # RECENT UPDATE: check that is correct # Update only those keys that are in soft deps for k in udeps: if k in depsx: udeps[k]=depsx[k] # udeps.update(depsx) preset_deps = i.get('preset_deps', {}) dep_add_tags = i.get('dep_add_tags', {}) for q in i: if q.startswith('deps.'): preset_deps[q[5:]]=i[q].split(':')[-1] elif q.startswith('dep_add_tags.'): _ , dep_name = q.split('.') dep_add_tags[dep_name] = i[q] for q in preset_deps: if q in udeps: udeps[q]['uoa']=preset_deps[q] cus=d.get('customize',{}) env=d.get('env',{}) # This environment will be passed to process scripts (if any) pr_env=cus.get('install_env',{}) # Update this env from CK kernel (for example, to decide what to use, git or https). FIXME: check the desired precedence between cus and ck.cfg pr_env.update(ck.cfg.get('install_env',{})) # Some main dictionary keys have been previously living there, # but now that we have variations it is more practical to move them to 'customize' subdictionary # which can be more flexibly edited based on variations. # If you need this functionality, please add these keys into the following tuple: for moving_key in ('package_name', 'package_extra_name'): if (moving_key not in cus) and (moving_key in d): cus[moving_key]=d[moving_key] supported_variations = d.get('variations', {}) required_vari_pairs = [ (variation_name, False) for variation_name in required_variations ] default_vari_pairs = [ (variation_name, True) for variation_name in supported_variations if supported_variations[variation_name].get('on_by_default', '')=='yes' ] vari_pairs = required_vari_pairs + default_vari_pairs # NB: the order is important! # Update this env from all the supported variations. # Detect if an incompatible mix of variation tags was required # that would lead to undefined behaviour, and bail out if so. # if vari_pairs: extra_env_from_variations = {} extra_cus_from_variations = {} extra_tags_from_variations = [] supported_variations = d.get('variations', {}) for (curr_variation, optional_variation) in vari_pairs: extra_env = supported_variations[curr_variation].get('extra_env',{}) colliding_vars = set(extra_env_from_variations.keys()) & set(extra_env.keys()) # non-empty intersection means undefined behaviour if colliding_vars and optional_variation: # a non-critical collision of env means we simply skip the optional_variation continue for coll_var in colliding_vars: # have to check actual values to detect a mismatch if extra_env_from_variations[coll_var] != extra_env[coll_var]: return { 'return':1, 'error':'contradiction on variable ({}) detected when adding "{}" variation tag'.format(coll_var,curr_variation)} extra_cus = supported_variations[curr_variation].get('extra_customize',{}) colliding_cuss = set(extra_cus_from_variations.keys()) & set(extra_cus.keys()) # non-empty intersection means undefined behaviour if colliding_cuss and optional_variation: # a non-critical collision of cus means we simply skip the optional_variation continue for coll_cus in colliding_cuss: # have to check actual values to detect a mismatch if extra_cus_from_variations[coll_cus] != extra_env[coll_cus]: return { 'return':1, 'error':'contradiction on customize ({}) detected when adding "{}" variation tag'.format(coll_cus,curr_variation)} extra_var_tags = supported_variations[curr_variation].get('extra_tags',[]) if extra_var_tags and type(extra_var_tags)!=list: extra_var_tags=extra_var_tags.split(',') extra_env_from_variations.update( extra_env ) # merge of one particular variation extra_cus_from_variations.update( extra_cus ) extra_tags_from_variations.extend( [curr_variation] + extra_var_tags ) vari_and_cli_tags.add(curr_variation) pr_env.update( extra_env_from_variations ) # merge of all variations cus.update( extra_cus_from_variations ) tags.extend( extra_tags_from_variations ) # Customize keys that start with a '*' are requesting environment substitution. for raw_cus_key in [ k for k in cus.keys() if k[0]=='*' ]: raw_cus_value = cus.pop(raw_cus_key) rx = ck.access({'action':'substitute_from_dict', 'module_uoa': 'misc', 'input_string': raw_cus_value, 'mapping': pr_env, 'tolerant': 'no', }) if rx['return']>0: return rx cus_key = raw_cus_key[1:] cus[cus_key] = rx['output_string'] ck.out("customize substitution: [{}] '{}' -> '{}'".format(cus_key, raw_cus_value, cus[cus_key])) # Environment keys that start with a '*' are requesting environment substitution. for raw_env_key in [ k for k in pr_env.keys() if k[0]=='*' ]: raw_env_value = pr_env.pop(raw_env_key) rx = ck.access({'action':'substitute_from_dict', 'module_uoa': 'misc', 'input_string': raw_env_value, 'mapping': pr_env, 'tolerant': 'no', }) if rx['return']>0: return rx env_key = raw_env_key[1:] pr_env[env_key] = rx['output_string'] ck.out("install_env substitution: [{}] '{}' -> '{}'".format(env_key, raw_env_value, pr_env[env_key])) # Conservatively setting cus['version'] to PACKAGE_VERSION only if it was previously unset. # FIXME: review the desired precedence between 'version' and PACKAGE_VERSION , taking into account that # 1) 'version' could have been set either directly in cus or selected via supported_versions and # 2) PACKAGE_VERSION could have been set either in cus['install_env'] or via variations..['extra_env'] if not cus.get('version') and pr_env.get('PACKAGE_VERSION'): cus['version'] = pr_env.get('PACKAGE_VERSION') ver=cus.get('version', '') env_display_name=cus.get('package_name', '') extra_version=i.get('extra_version', cus.get('extra_version','') ) extra_dir=cus.get('extra_dir','') # Check if need to ask version - this is useful when # a package downloads specific file depending on the version # and it is also reflected in the installed path # (see GCC universal installation) if d.get('ask_version','')=='yes' and force_version=='': ck.out('') r=ck.inp({'text':'Enter version of the package you would like to install: '}) if r['return']>0: return r ver=r['string'].strip() # Force version if force_version!='': ver = cus['version'] = force_version pr_env['PACKAGE_VERSION']=ver ver += extra_version # extra_version is normally added for disambiguation on CK side, so we add it later to avoid spoiling PACKAGE_VERSION tags.append('host-os-'+hosx) tags.append('target-os-'+tosx) tags.append(tbits+'bits') enruoa=i.get('env_repo_uoa','') enduoa=i.get('env_data_uoa','') enduid=i.get('env_data_uid','') # Update this env from customize meta (for example to pass URL to download package) # pr_env.update(cus.get('install_env',{})) # I moved it up to record changed env! for kpe in pr_env: x=pr_env[kpe] if x==str(x): x=str(x).replace('$#sep#$',sdirs) j=x.find('$#path_to_cid=') if j>=0: j1=x.find('#$',j+13) if j1>0: xcid=x[j+14:j1] # Try to resolve CID rx=ck.access({'action':'find', 'cid':xcid}) if rx['return']>0: return {'return':rx['return'], 'error':'Can\'t find entry when processing install_env var "'+kpe+'" ('+rx['error']+')'} rxp=rx['path'] x=x[:j]+rxp+x[j1+2:] pr_env[kpe]=x ########################################################################################## # # All of the following: # (1) "original" directory of the current entry # (2) directory of the entry pointed by 'use_scripts_from_another_entry' # (3) directory of the entry pointed by 'use_preprocess_scripts_from_another_entry' # may have a customization python script 'custom.py' (named in module/package/.cm/meta.json) # # The interface methods that will be optionally called (if the script provides them) are: # * pre_path() # * post_deps() # * setup() # * post_setup() # # In case both the "original" and the "other" directory contain the customization script, # each of the methods listed above will be attempted: # * first in the "other" (typically representing the base entry) # * and then in the "original" (typically representing the derived entry) # # The top two entries listed above may also contain install.{sh|bat} scripts. # # The mechanism seems to be an attempt to perform multiple inheritance of the entries, # both 'use_scripts_from_another_entry' and 'use_preprocess_scripts_from_another_entry' # representing the parents. # ########################################################################################## customization_script=None original_customization_script=None # original - always in the current directory of a package! # Initial assumption is that the "original" directory of the entry contains the customization script: # (customization_script_path, shell_script_path) = (p, p) # Loading original_customization_script from the original entry, if possible: # rx=ck.load_module_from_path({'path':customization_script_path, 'module_code_name':cfg['custom_script_name'], 'skip_init':'yes'}) if rx['return']==0: original_customization_script=rx['code'] # The second entry may contain both custom.py and install.{sh|bat} : # another_entry_with_scripts=d.get('use_scripts_from_another_entry', None) if another_entry_with_scripts: r=ck.access({'action':'find', 'module_uoa': another_entry_with_scripts.get('module_uoa', work['self_module_uid']), 'data_uoa': another_entry_with_scripts.get('data_uoa','') }) if r['return']>0: return r (customization_script_path, shell_script_path) = (r['path'], r['path']) # may change later via use_preprocess_scripts_from_another_entry # The third entry may only contain custom.py : # another_entry_with_preprocess_scripts=d.get('use_preprocess_scripts_from_another_entry', None) if another_entry_with_preprocess_scripts: r=ck.access({'action':'find', 'module_uoa': another_entry_with_preprocess_scripts.get('module_uoa', work['self_module_uid']), 'data_uoa': another_entry_with_preprocess_scripts.get('data_uoa','') }) if r['return']>0: return r customization_script_path=r['path'] # What follows may be a bit confusing: # # If original_customization_script existed, we will only attempt to load customization_script from another_entry_with_preprocess_scripts. # # However if there was no original_customization_script, we may attempt to load: # (1) original_customization_script again [ if defined last ] - will silently fail again # (2) customization_script from use_scripts_from_another_entry [ if defined last ] # (3) customization_script from use_preprocess_scripts_from_another_entry [ if defined last ] # In any case the loading is allowed to silently fail without notice. # if another_entry_with_preprocess_scripts or not original_customization_script: rx=ck.load_module_from_path({'path':customization_script_path, 'module_code_name':cfg['custom_script_name'], 'skip_init':'yes'}) if rx['return']==0: customization_script=rx['code'] else: # if there was no other script to load, we move the previously loaded code into customization_script (original_customization_script, customization_script) = (None, original_customization_script) # Check if need host CPU params features={} if d.get('need_cpu_info','')=='yes': r=ck.access({'action':'detect', 'module_uoa':cfg['module_deps']['platform.cpu'], 'host_os':hos, 'target_os':hos}) if r['return']>0: return r cpu_ft=r.get('features',{}).get('cpu',{}) features.update(r.get('features',{})) pr_env['CK_HOST_CPU_NUMBER_OF_PROCESSORS']=cpu_ft.get('num_proc','1') # We may want to pass more info (including target CPU) ... # Set up extra vars pr_env['CK_TARGET_CPU_BITS']=tosd.get('bits','') pr_env['CK_HOST_OS_ID']=hosn pr_env['CK_TARGET_OS_ID']=osn pr_env['CK_MD5SUM_CMD']=hosd.get('md5sum', 'md5sum') # Check if need host GPGPU params # We need a question here ('out':oo), since there can be multiple available drivers and we need to let user select the right one if d.get('need_gpgpu_info','')=='yes': r=ck.access({'action':'detect', 'module_uoa':cfg['module_deps']['platform.gpgpu'], 'type':d.get('need_gpgpu_type',''), 'host_os':hos, 'target_os':hos, 'out':oo}) if r['return']>0: return r features.update(r.get('features',{})) # Update env from input envx=i.get('env',{}) for q in i: if q.startswith('env.'): envx[q[4:]]=i[q] if len(envx)>0: pr_env.update(envx) # Search by exact terms setup={'host_os_uoa':hos, 'target_os_uoa':tos, 'target_os_bits':tbits} if ver!='': setup['version']=ver # Resolve deps if cus.get('ignore_deps','')=='yes': udeps={} sdeps='' if len(udeps)>0: env_resolve_action_dict={ 'action':'resolve', 'module_uoa':cfg['module_deps']['env'], 'host_os':hos, 'target_os':tos, 'target_device_id':tdid, 'repo_uoa':enruoa, 'install_to_env':iev, 'install_env':pr_env, 'reuse_deps':reuse_deps, 'deps_cache':deps_cache, 'dep_add_tags': dep_add_tags, 'safe':safe, 'deps':udeps} if o=='con': env_resolve_action_dict['out']='con' rx=ck.access(env_resolve_action_dict) if rx['return']>0: return rx sdeps=rx['bat'] udeps=rx['deps'] # Update deps (add UOA) for q in udeps: v=udeps[q] if v.get('uoa','')!='': setup['deps_'+q]=v['uoa'] # Check installation path pre_path=i.get('path','') pi=i.get('install_path','') vari_and_cli_path = '-'.join([''] + sorted(list(vari_and_cli_tags)) ) if vari_and_cli_tags else '' extra_path=i.get('extra_path','') fp=i.get('full_path','') x=cus.get('input_path_example','') if x!='': pie=' (example: '+x+')' else: pie='' # If rebuild option, try to set vars to avoid download if rebuild=='yes': pr_env.update(fix_env_for_rebuild) # Customize installation before installation path is finalized ****************************************************** param_dict_for_pre_path={"host_os_uoa":hosx, "host_os_uid":hos, "host_os_dict":hosd, "target_os_uoa":tosx, "target_os_uid":tos, "target_os_dict":tosd, "target_device_id":tdid, "cfg":d, "tags":tags, "env":env, "install_env":pr_env, "deps":udeps, "customize":cus, "self_cfg":cfg, "features":features, "version":ver, "ck_kernel":ck } if o=='con': param_dict_for_pre_path['interactive']='yes' if i.get('quiet','')=='yes': param_dict_for_pre_path['interactive']='' rx = internal_run_if_present(customization_script, 'pre_path', param_dict_for_pre_path, pr_env) if rx['return']>0: return rx rx = internal_run_if_present(original_customization_script, 'pre_path', param_dict_for_pre_path, pr_env) if rx['return']>0: return rx dep_tags = [] # Iterate through all resolved dependencies and check which tags we need to create from them, # preserving the desired sort order: # for dep_name, dep_dict in sorted(udeps.items(), key=lambda pair: pair[1].get('sort',0)) : if dep_dict.get('uoa',''): if dep_name in ('compiler', 'host-compiler') : dep_tag_prefix = 'compiled-by-' elif dep_dict.get('add_to_tags', dep_dict.get('add_to_path','') )=='yes': dep_tag_prefix = 'needs-' else: dep_tag_prefix = '' # Empty prefix means we don't want this dependency to appear in tags: # if dep_tag_prefix: dep_tag = dep_tag_prefix + dep_dict.get('build_dir_name','unknown_' + dep_name) dep_tags.append( dep_tag ) dep_version = dep_dict.get('ver') if dep_version: dep_tags.append( dep_tag + '-' + dep_version ) # Join stripped tags and compiler tags into a CSV string: stripped_tags = [t.strip() for t in tags if t.strip()] tags_csv = ','.join( dep_tags + stripped_tags ) xprocess = i.get('skip_process','')!='yes' or rebuild=='yes' or reinstall=='yes' xsetup = i.get('skip_setup','')!='yes' and d.get('skip_setup','')!='yes' shell_script_name=d.get('process_script','') if pi=='': # Check if environment already exists to check installation path if enduoa=='': if o=='con': ck.out('') ck.out('Searching if CK environment for this package already exists using:') ck.out(' * Tags: '+tags_csv) if len(udeps)>0: for q in udeps: v=udeps[q] vuoa=v.get('uoa','') if vuoa!='': ck.out(' * Dependency: '+q+'='+v.get('uoa','')) r=ck.access({'action':'search', 'module_uoa':cfg['module_deps']['env'], 'tags':tags_csv, 'search_dict':{'setup':setup}}) if r['return']>0: return r lst=r['lst'] # If more than one entry, try to prune by package UID if exists if len(lst)>1 and duid!='': new_lst=[] for je in lst: skip=False rje=ck.access({'action':'load', 'module_uoa':cfg['module_deps']['env'], 'data_uoa':je['data_uid'], 'repo_uoa':je['repo_uid']}) if rje['return']==0: # print (rje['dict'].get('customize',{}).get('used_package_uid','')) # print (duid) if rje['dict'].get('customize',{}).get('used_package_uid','')!=duid: skip=True if not skip: new_lst.append(je) lst=new_lst if len(lst)==1: fe=lst[0] enruoa=fe['repo_uid'] enduoa=fe['data_uoa'] enduid=fe['data_uid'] if iev=='yes': pi=fe['path'] if o=='con': x=enduoa if enduid!=enduoa: x+=' ('+enduid+')' ck.out('') ck.out('CK environment found for this package: '+x) elif len(lst)>1: ck.out('') ck.out('AMBIGUITY: more than one environment entry found for this installation') ck.out('') for je in lst: ck.out(' * '+je['data_uid']) ck.out('') return {'return':1, 'error':'more than one environment entry found for this installation, please specify using --env_data_uoa={correct environment entry UID}'} else: if o=='con': ck.out('') ck.out('CK environment not found for this package ...') # Load env if exists if enduoa!='': r=ck.access({'action':'load', 'module_uoa':cfg['module_deps']['env'], 'repo_uoa':enruoa, 'data_uoa':enduoa}) if r['return']>0: return r de=r['dict'] x=de.get('customize',{}).get('path_install','') if x!='': pi=x x=de.get('customize',{}).get('full_path','') if x!='': fp=x # if extra_dir!='': # j=pi.rfind(extra_dir) # if j>=0: # pi=pi[:j] # # if pi!='': # j=len(pi) # if pi[j-1]==sdirs: # pi=pi[:-1] if fp!='': if o=='con': if ask=='yes' and xprocess: ck.out('') ck.out('It appears that package is already installed or at least file from the package is already found in path: '+fp) if shell_script_name: ck.out('') rx=ck.inp({'text':'Would you like to overwrite and process it again (y/N)? '}) x=rx['string'].strip().lower() if x!='y' and x!='yes': xprocess=False if ask=='yes' and xsetup: ck.out('') rx=ck.inp({'text':'Would you like to setup environment for this package again (Y/n)? '}) x=rx['string'].strip().lower() if x=='n' or x=='no': xsetup=False if ask!='yes' and xprocess: ck.out('') ck.out(' OVERWRITING AND PROCESSING AGAIN!') else: return {'return':1, 'error':'package is already installed in path '+pi} if enduoa=='' and iev=='yes': # Create dummy env and then set path there # TBD - if installation fails, we still have this dummy - need to check what to do ... # can remove: ck rm env:* --tags=tmp xx=tags_csv if xx!='': xx+=',' xx+='tmp' rx=ck.access({'action':'add', 'module_uoa':cfg['module_deps']['env'], 'repo_uoa':enruoa, 'tags':xx, 'dict':{'setup':setup}}) if rx['return']>0: return rx enduoa=rx['data_uoa'] enduid=rx['data_uid'] pi=rx['path'] if cus.get('skip_path','')!='yes' and pi=='': if o=='con': ck.out('') pix='' sp=d.get('suggested_path','') if pre_path=='': rz=prepare_install_path({}) if rz['return']>0: return rz x=rz['path'] else: x=pre_path if x!='' and sp!='': # Prepare installation path # First via package + version nm=sp # if cus.get('no_ver_in_suggested_path','')!='yes' and cus.get('version','')!='': # nm+='-'+cus.get('version','') if cus.get('no_ver_in_suggested_path','')!='yes' and ver!='': nm+='-'+ver.strip('-') # Then if compiler bdn=udeps.get('compiler',{}).get('build_dir_name','') vr=udeps.get('compiler',{}).get('ver','') if bdn=='': bdn=udeps.get('support_compiler',{}).get('build_dir_name','') vr=udeps.get('support_compiler',{}).get('ver','') if cus.get('no_compiler_in_suggested_path','')!='yes' and bdn!='': nm+='-'+bdn if vr!='': nm+='-'+vr # Then any deps with explicitly specified 'add_to_path' for u in sorted(udeps, key=lambda v: udeps[v].get('sort',0)): uu=udeps[u] if uu.get('add_to_path','')=='yes': vr=uu.get('ver','') softuoa=uu.get('dict',{}).get('soft_uoa','') salias=uu.get('dict',{}).get('soft_alias','') if salias=='': salias=softuoa if salias!='': nm+='-'+salias if vr!='': nm+='-'+vr # Then some extra path, if non-empty nm += cus.get('extra_suggested_path','') # Then another extra path, if non-empty nm += vari_and_cli_path + extra_path # Finally OS if cus.get('no_os_in_suggested_path','')!='yes': nm+='-'+tosx pix=os.path.join(x, nm) if cus.get('no_os_in_suggested_path','')!='yes': if not tosx.endswith(tbits): pix+='-'+tbits if o=='con' and (ask=='yes' or cus.get('force_ask_path','')=='yes'): ck.out('*** Suggested installation path: '+pix) r=ck.inp({'text':' Press Enter to use suggested path or input new installation path '+pie+': '}) pi=r['string'].strip() if pi=='': pi=pix else: pi=pix if d.get('no_install_path','')!='yes': ck.out('*** Installation path used: '+pix) if o=='con': ck.out('') else: if o=='con': r=ck.inp({'text':'Enter installation path '+pie+': '}) pi=r['string'].strip() if pi=='' and cus.get('skip_path','')!='yes': return {'return':1, 'error':'installation path is not specified'} suoa=d.get('soft_uoa','') or cus.get('soft_uoa', '') # Check dependencies deps={} soft_dict={} if suoa!='': rx=ck.access({'action':'load', 'module_uoa':cfg['module_deps']['soft'], 'data_uoa':suoa}) if rx['return']>0: return rx soft_dict=rx['dict'] deps=soft_dict.get('deps',{}) # In case 'package_name' was missing from the package, use the default from soft entry if env_display_name=='': env_display_name=soft_dict.get('soft_name','') env_display_name+=cus.get('package_extra_name', '') # TODO: instead of this application-specific substitute, start using generic # "*package_extra_name" or "*package_name" substitute - then the following # paragraph could be removed: rx = ck.access({'action':'substitute_from_dict', 'module_uoa': 'misc', 'input_string': env_display_name, 'mapping': pr_env, }) if rx['return']>0: return rx env_display_name = rx['output_string'] # Save package UOA to the cus cus['used_package_uoa']=duoa cus['used_package_uid']=duid # Update by package deps (more precise) for q in deps: v=deps[q] if q not in udeps: udeps[q]=v # Prepare environment based on deps if cus.get('ignore_deps','')=='yes': udeps={} sdeps='' if len(udeps)>0: env_resolve_action_dict={'action':'resolve', 'module_uoa':cfg['module_deps']['env'], 'host_os':hos, 'target_os':tos, 'target_device_id':tdid, 'repo_uoa':enruoa, 'install_to_env':iev, 'reuse_deps':reuse_deps, 'deps_cache':deps_cache, 'dep_add_tags': dep_add_tags, 'safe':safe, 'deps':udeps} if rebuild=='yes': env_resolve_action_dict['rebuild']='yes' if o=='con': env_resolve_action_dict['out']='con' rx=ck.access(env_resolve_action_dict) if rx['return']>0: return rx sdeps=rx['bat'] if o=='con' and pi!='': ck.out('') ck.out('Installing to '+pi) ck.out('') # Customize installation based on resolved dependencies ************************************************************* param_dict_for_post_deps={"host_os_uoa":hosx, "host_os_uid":hos, "host_os_dict":hosd, "target_os_uoa":tosx, "target_os_uid":tos, "target_os_dict":tosd, "target_device_id":tdid, "cfg":d, "tags":tags, "env":env, "features":features, "customize":cus, "self_cfg":cfg, "version":ver, "ck_kernel":ck, "deps":udeps } if o=='con': param_dict_for_post_deps['interactive']='yes' if i.get('quiet','')=='yes': param_dict_for_post_deps['interactive']='' rx = internal_run_if_present(customization_script, 'post_deps', param_dict_for_post_deps, pr_env) if rx['return']>0: return rx rx = internal_run_if_present(original_customization_script, 'post_deps', param_dict_for_post_deps, pr_env) if rx['return']>0: return rx hd_r=ck.access({'action': 'get_home_dir', 'module_uoa': 'misc', }) if hd_r['return']>0: return r home_dir=hd_r['home_dir'] soft_cfg={} # Check if continue processing if (shell_script_name or (customization_script and 'setup' in dir(customization_script))) and xprocess: # start bat shell_wrapper_contents=hosd.get('batch_prefix','')+'\n' if host_add_path_string!='': shell_wrapper_contents+=host_add_path_string+'\n\n' # Check if extra params to pass as environment param=i.get('param',None) params=d.get('params',{}) params.update(i.get('params',{})) # Parse -D ... for k in i: if k.startswith('D'): params[k[1:]]=i[k] if param!=None: shell_wrapper_contents+='\n' xs='' if param.find(' ')>=0 and eifs!='': xs=eifs shell_wrapper_contents+=eset+' CK_PARAM='+xs+param+xs+'\n' if len(params)>0: for q in params: v=params[q] if v!=None: xs='' if v.find(' ')>=0 and eifs!='': xs=eifs shell_wrapper_contents+=eset+' '+q+'='+xs+v+xs+'\n' shell_wrapper_contents+='\n' # Check installation path if pi=='' and cus.get('skip_path','')!='yes': if o=='con': ck.out('') pix='' sp=d.get('suggested_path','') rz=prepare_install_path({}) if rz['return']>0: return rz x=rz['path'] if x!='' and sp!='': pix=os.path.join(x, sp+'-'+cus.get('version','')+'-'+tosx) if not tosx.endswith(tbits): pix+='-'+tbits ck.out('Suggested path: '+pix) r=ck.inp({'text':' Press Enter to use suggested path or input new installation path '+pie+': '}) pi=r['string'].strip() if pi=='': pi=pix else: r=ck.inp({'text':'Enter installation path '+pie+': '}) pi=r['string'].strip() if pi=='': return {'return':1, 'error':'installation path is not specified'} def efp_substitute(efp): efp=efp.replace('$#home#$', home_dir) efp=efp.replace('$#sep#$', sdirs) efp=efp.replace('$#abi#$', tosd.get('abi','')) efp=efp.replace('$#processor#$', tosd.get('processor','')) for k in pr_env: v=str(pr_env[k]) efp=efp.replace('$<<' +k+ '>>$',v) # NOTE: adapted from module/soft/module.py/prepare_target_name() # After successful testing this function should be moved out # into a common utility space and imported from there. # Also, check whether the original function was supposed to be # sourcing the data from hosd or tosd. file_extensions=tosd.get('file_extensions',{}) for k in file_extensions: v=file_extensions[k] efp=efp.replace('$#file_ext_'+k+'#$',v) host_file_extensions=hosd.get('host_file_extensions',{}) for k in host_file_extensions: v=host_file_extensions[k] efp=efp.replace('$#host_file_ext_'+k+'#$',v) return efp # Check if there is already library or tool exists efp=d.get('end_full_path_universal') if efp==None or len(efp)==0: efpd = d.get('end_full_path', {}) efp = efpd['macos'] if (macos and 'macos' in efpd) else efpd.get(tname2,[]) efp_or_list = efp if type(efp)==list else [ efp ] # enforce list fp_candidates = [ os.path.join( pi, efp_substitute(x) ) for x in efp_or_list ] # apply substitution and create full paths fp_matches = [ x for x in fp_candidates if os.path.exists(x) ] # find all matches cont=True if len(fp_matches)>0: if o=='con': ck.out('') ck.out('It appears that package is already installed or at least file from the package already found in path: {}'.format(fp_matches)) if (rebuild!='yes' and reinstall!='yes') or ask=='yes': ck.out('') rx=ck.inp({'text':'Would you like to overwrite/process it again (y/N)? '}) x=rx['string'].strip().lower() if x!='y' and x!='yes': cont=False else: ck.out('') ck.out(' OVERWRITING AND PROCESSING AGAIN!') # Check if need to use scripts from another entry if cont: # Customize main installation param_dict_for_setup={"host_os_uoa":hosx, "host_os_uid":hos, "host_os_dict":hosd, "target_os_uoa":tosx, "target_os_uid":tos, "target_os_dict":tosd, "target_device_id":tdid, "cfg":d, "tags":tags, "env":env, "new_env":pr_env, "deps":udeps, "features":features, "customize":cus, "self_cfg":cfg, "version":ver, "path":shell_script_path, "path_original_package":p, "script_path":customization_script_path, "out":oo, "install_path":pi } if o=='con': param_dict_for_setup['interactive']='yes' if i.get('quiet','')=='yes': param_dict_for_setup['interactive']='' param_dict_for_post_setup=copy.deepcopy(param_dict_for_setup) param_dict_for_post_setup['ck_kernel']=ck param_dict_for_setup['ck_kernel']=ck rx = internal_run_if_present(customization_script, 'setup', param_dict_for_setup, pr_env) if rx['return']>0: return rx else: soft_cfg=rx.get('soft_cfg',{}) rx = internal_run_if_present(original_customization_script, 'setup', param_dict_for_setup, pr_env) if rx['return']>0: return rx # Prepare process script if shell_script_name: shell_script_name+=sext shell_script_full_path=os.path.join(shell_script_path,shell_script_name) if not os.path.isfile(shell_script_full_path): return {'return':1, 'error':'processing script '+shell_script_name+' is not found'} # Add deps if needed before running if sdeps!='': shell_wrapper_contents+=sdeps # Add compiler dep again, if there shell_wrapper_contents+='\n' for k in sorted(udeps, key=lambda v: udeps[v].get('sort',0)): if 'compiler' in k or udeps[k].get('repeat_at_the_end','')=='yes': x=udeps[k].get('bat','').strip() if x!='' and not shell_wrapper_contents.endswith(x): shell_wrapper_contents+=x+' 1\n' shell_wrapper_contents+='\n' # Add misc environment (prepared above) for q in pr_env: qq=str(pr_env[q]) qq=qq.replace('$<<',svarb).replace('>>$',svare) if qq.find(' ')>0: qq=eifs+qq+eifs shell_wrapper_contents+=eset+' '+q+'='+qq+'\n' # If install path has space, add quotes for some OS ... xs='' if pi.find(' ')>=0 and eifs!='': xs=eifs shell_wrapper_contents+=eset+' INSTALL_DIR='+xs+pi+xs+'\n' # If Windows, add MingW path if wb=='yes': rm=ck.access({'action':'convert_to_cygwin_path', 'module_uoa':cfg['module_deps']['os'], 'path':pi}) if rm['return']>0: return rm ming_pi=rm['path'] shell_wrapper_contents+=eset+' INSTALL_DIR_MINGW='+xs+ming_pi+xs+'\n' xs='' if p.find(' ')>=0 and eifs!='': xs=eifs shell_wrapper_contents+=eset+' PACKAGE_DIR='+xs+shell_script_path+xs+'\n' # If Windows, add MingW path if wb=='yes': rm=ck.access({'action':'convert_to_cygwin_path', 'module_uoa':cfg['module_deps']['os'], 'path':shell_script_path}) if rm['return']>0: return rm ming_ppp=rm['path'] shell_wrapper_contents+=eset+' PACKAGE_DIR_MINGW='+xs+ming_ppp+xs+'\n' xs='' if p.find(' ')>=0 and eifs!='': xs=eifs shell_wrapper_contents+=eset+' ORIGINAL_PACKAGE_DIR='+xs+p+xs+'\n' # If Windows, add MingW path if wb=='yes': rm=ck.access({'action':'convert_to_cygwin_path', 'module_uoa':cfg['module_deps']['os'], 'path':p}) if rm['return']>0: return rm ming_p=rm['path'] shell_wrapper_contents+=eset+' ORIGINAL_PACKAGE_DIR_MINGW='+xs+ming_p+xs+'\n' shell_wrapper_contents+='\n' if debug=='yes': import platform xs='Warning: you are in a new shell with a pre-set CK environment. Enter "exit" to return to the original one!' if platform.system().lower().startswith('win'): shell_wrapper_contents+='echo '+xs+'\n' shell_wrapper_contents+='cmd\n' else: shell_wrapper_contents+='echo "'+xs+'"\n' shell_wrapper_contents+='bash\n' xs='' if p.find(' ')>=0 and eifsc!='': xs=eifsc shell_wrapper_contents+=scall+' '+xs+shell_script_full_path+xs+'\n\n' if wb=='yes' and d.get('check_exit_status','')!='yes': shell_wrapper_contents+='exit /b 0\n' rs=i.get('record_script','') # Generate tmp file (or use record script) if rs: shell_wrapper_name=rs if shell_wrapper_name==os.path.basename(shell_wrapper_name): shell_wrapper_name=os.path.join(os.getcwd(),shell_wrapper_name) else: rx=ck.gen_tmp_file({'prefix':'tmp-ck-', 'suffix':sext}) if rx['return']>0: return rx shell_wrapper_name=rx['file_name'] # Write to tmp file rx=ck.save_text_file({'text_file':shell_wrapper_name, 'string':shell_wrapper_contents}) if rx['return']>0: return rx # Go to installation path if not os.path.isdir(pi): os.makedirs(pi) os.chdir(pi) # Check if need to set executable flags se=hosd.get('set_executable','') if se!='': x=se+' '+shell_wrapper_name rx=os.system(x) # Run script rx=os.system(shell_wrapper_name) # Remove script (if tmp) if rs=='' and os.path.isfile(shell_wrapper_name): os.remove(shell_wrapper_name) if rx>0: print_warning({'package_data_uoa':duoa, 'package_repo_uoa':package_repo_uoa, 'package_meta':d}) return {'return':1, 'error':'package installation failed'} # Check if has post-setup Python script param_dict_for_post_setup['new_env']=pr_env rx = internal_run_if_present(customization_script, 'post_setup', param_dict_for_post_setup, {}) if rx['return']>0: return rx rx = internal_run_if_present(original_customization_script, 'post_setup', param_dict_for_post_setup, {}) if rx['return']>0: return rx fp_matches = [ x for x in fp_candidates if os.path.exists(x) ] # find all matches again fp = fp_matches[0] if len(fp_matches) else fp_candidates[0] if len(fp_candidates) else pi if 'env_prefix' in i: cus['env_prefix'] = i['env_prefix'] # Preparing soft registration soft_registration_action_dict={'action':'setup', 'module_uoa':cfg['module_deps']['soft'], 'data_uoa':suoa, 'soft_name':env_display_name, 'host_os':hos, 'target_os':tos, 'target_device_id':tdid, 'tags':tags_csv, 'customize':cus, 'features':features, 'env_new':'yes', 'env_repo_uoa':enruoa, 'env_data_uoa':enduoa, 'env':env, 'extra_version':extra_version } nw='no' if enduoa=='': nw='yes' if cus.get('collect_device_info','')!='yes': soft_registration_action_dict['skip_device_info_collection']='yes' if d.get('remove_deps','')=='yes': soft_registration_action_dict['deps_copy']=udeps else: soft_registration_action_dict['deps']=udeps if d.get('no_install_path','')!='yes': if fp!='': soft_registration_action_dict['full_path']=fp soft_registration_action_dict['full_path_install']=pi elif pi!='': # mainly for compatibility with previous CK soft manager soft_registration_action_dict['install_path']=pi if duid!='': soft_registration_action_dict['package_uoa']=duid if len(soft_cfg)>0: soft_registration_action_dict.update(soft_cfg) ck_install_file_contents=copy.deepcopy(soft_registration_action_dict) # Check if need to setup environment if xsetup: if suoa=='': return {'return':1, 'error':'Software environment UOA is not defined in this package (soft_uoa)'} if suoa!='': if o=='con': ck.out('') ck.out('Setting up environment for installed package ...') ck.out(' (full path = '+fp+')') ck.out('') if o=='con': soft_registration_action_dict['out']='con' rx=ck.access(soft_registration_action_dict) if rx['return']>0: return rx enduoa=rx['env_data_uoa'] enduid=rx['env_data_uid'] # Recording cus dict to install dir to be able to rebuild env later if needed if pi!='': ck_install_file_path=os.path.join(pi, cfg['ck_install_file']) if o=='con': ck.out('') ck.out('Recording CK configuration to '+ck_install_file_path+' ...') ck_install_file_contents['env_data_uoa']=enduid rx=ck.save_json_to_file({'json_file':ck_install_file_path, 'dict':ck_install_file_contents, 'sort_keys':'yes'}) if rx['return']>0: return rx if o=='con' and pi!='': ck.out('') ck.out('Installation path: '+pi) ck.out('') elapsed_time=time.time()-start_time if o=='con': ck.out('Installation time: '+str(elapsed_time)+' sec.') return {'return':0, 'elapsed_time':elapsed_time, 'env_data_uoa':enduoa, 'env_data_uid':enduid} ################################################################################## # internal function: run a method of a given customization script with param_dict # and top up the topup_from_install_env dictionary from install_env def internal_run_if_present(script, method_name, param_dict, topup_from_install_env): if script and method_name in dir(script): method = getattr(script, method_name) rx = method( param_dict ) if rx.get('return')==0: topup_from_install_env.update( rx.get('install_env',{}) ) else: rx = { 'return' : 0 }; return rx ############################################################################## # setup package (only environment) def setup(i): """ See "install" API with skip_process=yes """ i['skip_process']='yes' return install(i) ############################################################################## # internal function: get value from list without error if out of bounds def internal_get_val(lst, index, default_value): v=default_value if index 0, if error (error) - error text if return > 0 } """ o=i.get('out','') oo='' if o=='con': oo=o # Check if target if i.get('target','')!='': r=ck.access({'action':'init', 'module_uoa':cfg['module_deps']['machine'], 'input':i}) if r['return']>0: return r device_cfg=i.get('device_cfg',{}) # Check host/target OS/CPU hos=i.get('host_os','') tos=i.get('target_os','') tdid=i.get('target_device_id','') r=ck.access({'action':'detect', 'module_uoa':cfg['module_deps']['platform.os'], 'host_os':hos, 'target_os':tos, 'device_cfg':device_cfg, 'target_device_id':tdid, 'skip_info_collection':'yes'}) if r['return']>0: return r hos=r['host_os_uid'] hosx=r['host_os_uoa'] hosd=r['host_os_dict'] tos=r['os_uid'] tosx=r['os_uoa'] tosd=r['os_dict'] # Get list of deps duoa=i.get('data_uoa','') r=ck.access({'action':'load', 'module_uoa':work['self_module_uid'], 'data_uoa':duoa}) if r['return']>0: return r d=r['dict'] deps=d.get('deps',{}) for k in sorted(deps, key=lambda v: deps[v].get('sort',0)): dd=deps[k] dn=dd.get('name','') if dn=='': dn=k else: dn+=' ('+k+')' tags=dd.get('tags','') if oo=='con': ck.out('*******************************************************') ck.out('Dependency: '+dn) ck.out('Tags: '+tags) ck.out('') ntos=tos if dd.get('force_target_as_host','')=='yes': ntos=hos # Attempt to install package by tags r=ck.access({'action':'install', 'module_uoa':work['self_module_uid'], 'tags':tags, 'host_os':hos, 'target_os':ntos, 'device_id':tdid, 'out':oo}) if r['return']>0: ck.out('') ck.out('Package installation failed: '+r['error']+'!') return {'return':0} ############################################################################## # show available packages def show(i): """ Input: { (the same as list; can use wildcards) (out_file) - output to file (for mediawiki) } Output: { return - return code = 0, if successful > 0, if error (error) - error text if return > 0 } """ import os import copy o=i.get('out','') of=i.get('out_file','') if of!='': xof=os.path.splitext(of) html=False if o=='html' or i.get('web','')=='yes': html=True h='' h2='' if i.get('new','')=='yes': ii=copy.deepcopy(i) ii['action']='preload_html_for_lists' ii['module_uoa']=cfg['module_deps']['misc'] ii['ck_title']='Shared CK packages' r=ck.access(ii) if r['return']>0: return r h=r['html_start']+'\n' h2=r['html_stop']+'\n' unique_repo=False if i.get('repo_uoa','')!='': unique_repo=True list_action_dict=copy.deepcopy(i) list_action_dict['out']='' list_action_dict['action']='list' list_action_dict['add_meta']='yes' list_action_dict['time_out']=-1 rx=ck.access(list_action_dict) if rx['return']>0: return rx ll=sorted(rx['lst'], key=lambda k: k['data_uoa']) if html: h+='

Please check our new beta browser for CK components!

\n' h+='
\n' h+='

CK package manager unifies installation of code, data and models across different platforms and operating systems\n' h+='using the most appropriate solutions including wget, make, cmake, scons, spack, vcpkg, conan, easybuild, etc.\n' h+='

' h+='You can install a CK package into CK virtual environment as follows:\n' h+='

\n'
       h+=' ck pull repo:{Repo UOA - see below}\n'
       h+=' ck install package:{Package UOA - see below}\n'
       h+='
\n' h+='using tags:\n' h+='
\n'
       h+=' ck install package --tags={some tags from below}\n'
       h+='
\n' h+='to an unusual path:\n' h+='
\n'
       h+=' ck install package:{Package UOA - see below} --install_path={full installation path}\n'
       h+='
\n' h+='to a CK virtual environment entry :\n' h+='
\n'
       h+=' $ ck set kernel var.install_to_env=yes\n'
       h+=' ck install package:{Package UOA - see below}\n'
       h+='
\n' h+='or for a different OS target (Android):\n' h+='
\n'
       h+=' ck ls os:android* | sort\n'
       h+=' ck install package:{Package UOA - see below} --target_os={OS UOA from above}\n'
       h+='
\n' h+='You can then see or use registered virtual CK environments for installed packages as follows:\n' h+='
\n'
       h+=' ck show env\n'
       h+=' ck show env --tags={some tags from below}\n'
       h+='\n'
       h+=' ck virtual env:{UID from above}\n'
       h+=' ck virtual env --tags={some tags from below}\n'
       h+='
\n' h+='

\n' h+='See 

ck install package --help
for more installation options.\n' h+='See related CK soft detection plugins,\n' h+=' CK documentation,\n' h+=' "how to contribute" guide,\n' h+=' ACM ReQuEST-ASPLOS\'18 report\n' h+=' and the latest CK paper for further details.\n' h+='See ck-spack repository connecting CK and spack package manager.\n' h+='

\n' h+='\n' h+=' \n' h+=' \n' h+=' \n' h+=' \n' h+=' \n' h+=' \n' h+=' \n' h+=' \n' h+=' \n' h+=' \n' repo_url={} repo_private={} size=0 isize=1 private='' num=0 for l in ll: ln=l['data_uoa'] lr=l['repo_uoa'] lr_uid=l['repo_uid'] url='' if lr=='default': url='' #'http://github.com/ctuning/ck' elif lr_uid in repo_url: url=repo_url[lr_uid] else: rx=ck.load_repo_info_from_cache({'repo_uoa':lr_uid}) if rx['return']>0: return rx url=rx.get('dict',{}).get('url','') repo_private[lr_uid]=rx.get('dict',{}).get('private','') repo_url[lr_uid]=url private=repo_private.get(lr_uid,'') if lr not in cfg.get('skip_repos',[]) and private!='yes' and url!='' and lr!='ck-spack': num+=1 lm=l['meta'] ld=lm.get('desc','') name=lm.get('soft_name','') cus=lm.get('customize',{}) ver=cus.get('version','') xhos=lm.get('only_for_host_os_tags',[]) xtos=lm.get('only_for_target_os_tags',[]) tmpl=lm.get('template','') template=lm.get('template_type','') if tmpl=='yes' and template=='': template='yes' tags=lm.get('tags',[]) ytags=','.join(tags) yhos='' ytos='' for q in xhos: if yhos!='': yhos+=',' yhos+=q for q in xtos: if ytos!='': ytos+=',' ytos+=q if yhos=='': yhos='any' else: yhos=yhos.replace('linux','linux,macos') if ytos=='': ytos='any' else: ytos=ytos.replace('linux','linux,macos') if lr=='default': to_get='' elif url.find('github.com/ctuning/')>0: to_get='ck pull repo:'+lr else: to_get='ck pull repo --url='+url x=lr y='' yh='' if url!='': url2=url if url2.endswith('.git'): url2=url2[:-4] yh=url2+'/tree/master/package/'+ln x='['+url2+' '+lr+']' y='['+yh+' link]' ############################################################### if html: h+=' \n' x1='' x2='' z1='' z11='' if url!='': x1='' x2='' z1='' z11='' h+=' \n' h+=' \n' h+=' \n' h+=' \n' h+=' \n' h+=' \n' ############################################################### elif o=='mediawiki': s='' x=lr y='' if url!='': x='['+url+' '+lr+']' y='['+url+'/tree/master/package/'+ln+' link]' s='\n' s+='=== '+ln+' ('+ver+') ===\n' s+='\n' s+='Tags: '+ytags+'\n' s+='
Host OS tags: '+yhos+'\n' s+='
Target OS tags: '+ytos+'\n' if y!='': s+='\n' s+='Package entry with meta: '+y+'\n' s+='\n' s+='Which CK repo: '+x+'\n' if to_get!='': s+='
How to get: '+to_get+'\n' if to_get!='': s+='\n' s+='How to install: ck install package:'+ln+' (--target_os={CK OS UOA})' s+='\n' if of=='': ck.out(s) else: with open(of, "a") as ff: ff.write(s) size+=len(s) if size>=100000: isize+=1 of=xof[0]+str(isize)+xof[1] size=0 ############################################################### elif o=='con' or o=='txt': if unique_repo: ck.out('') s=ln+' - '+ld else: ss='' if len(ln)<35: ss=' '*(35-len(ln)) ss1='' if len(lr)<30: ss1=' '*(30-len(lr)) s=ln+ss+' ('+lr+')' if ld!='': s+=ss1+' '+ld ck.out(s) ck.out('') ck.out(' Total packages: '+str(num)) ck.out('') if html: h+='
#Package UOATemplate?Repo UOATagsHost OSTarget OSNotes
'+str(num)+''+z1+ln+x2+' ('+z11+'CK meta'+x2+')'+template+''+x1+lr+x2+''+ytags+'\n' h+=' '+yhos+'\n' h+=' '+ytos+'\n' h1='' if ld!='': h1+='

\n'+ld h+='

'+h1+'\n' h+='
\n' h+=h2 if of!='': r=ck.save_text_file({'text_file':of, 'string':h}) if r['return']>0: return r return {'return':0, 'html':h} ############################################################################## # prepare installation path def prepare_install_path(i): """ Input: { } Output: { return - return code = 0, if successful > 0, if error (error) - error text if return > 0 path - prepared path where to install packages } """ import os # Moved Tools to $HOME by default if CK_TOOLS is not defined x=os.environ.get(cfg["env_install_path"],'') if x=='': # Get home user directory from os.path import expanduser home = expanduser("~") x=os.path.join(home, cfg["install_path"]) if not os.path.isdir(x): os.makedirs(x) return {'return':0, 'path':x} ############################################################################## # prepare distribution def distribute(i): """ Input: { (data_uoa) - package UOA (ext) - output package ext (ck-distro-{ext}).zip If not specified, UID is generated. or (filename) - full name of output package (path_key) - (path_bin, path_lib, path_include) } Output: { return - return code = 0, if successful > 0, if error (error) - error text if return > 0 } """ import os import shutil # Init o=i.get('out','') oo='' if o=='con': oo=o cur_dir=os.getcwd() duoa=i.get('data_uoa','') tags=i.get('tags','') pk=i.get('path_key','') if pk=='': pk='path_bin' # Extension fn=i.get('filename','') if fn=='': ext=i.get('ext','') if ext=='': rx=ck.gen_uid({}) if rx['return']>0: return rx ext=rx['data_uid'] fn='ck-distro-'+ext+'.zip' # Check if target if i.get('target','')!='': r=ck.access({'action':'init', 'module_uoa':cfg['module_deps']['machine'], 'input':i}) if r['return']>0: return r device_cfg=i.get('device_cfg',{}) # Check host/target OS/CPU hos=i.get('host_os','') tos=i.get('target_os','') tdid=i.get('target_device_id','') r=ck.access({'action':'detect', 'module_uoa':cfg['module_deps']['platform.os'], 'host_os':hos, 'target_os':tos, 'device_cfg':device_cfg, 'target_device_id':tdid, 'skip_info_collection':'yes'}) if r['return']>0: return r hos=r['host_os_uid'] hosx=r['host_os_uoa'] hosd=r['host_os_dict'] tos=r['os_uid'] tosx=r['os_uoa'] tosd=r['os_dict'] host_add_path_string=r.get('host_add_path_string','') hosn=hosd.get('ck_name2','') osn=tosd.get('ck_name2','') # Load package meta r=ck.access({'action':'load', 'module_uoa':work['self_module_uid'], 'data_uoa':duoa}) if r['return']>0: return r d=r['dict'] p0=r['path'] # Get tags tags=d['tags'] xtags=','.join(tags) # Resolve env if o=='con': ck.out('Searching environment for this package ...') r=ck.access({'action':'set', 'module_uoa':cfg['module_deps']['env'], 'tags':xtags, 'host_os':hos, 'target_os':tos, 'device_id':tdid, 'out':oo}) if r['return']>0: return r de=r['dict'] deps=de.get('deps',{}) # Path cus=de.get('customize',{}) pp=cus.get(pk,'') pp1=pp found=False while not found: pp1=os.path.dirname(pp) if pp1==pp: break ppx=os.path.join(pp1, cfg['ck_install_file']) if os.path.isfile(ppx): found=True break pp=pp1 # Read ck-install.json if not found: return {'return':1, 'error':pp1+' not found in installation paths ...'} r=ck.load_json_file({'json_file':ppx}) if r['return']>0: return r dx=r['dict'] # Rename deps (to avoid mix ups with local env) # dx['saved_deps']=dx.pop('deps') # Save to tmp file rx=ck.gen_tmp_file({'prefix':'tmp-ck-', 'suffix':'.json'}) if rx['return']>0: return rx ftmp=rx['file_name'] r=ck.save_json_to_file({'json_file':ftmp, 'dict':dx, 'sort_keys':'yes'}) if r['return']>0: return r # Check extra commands (for a given target platform) ec=d.get('distribute_extra_commands',{}).get(osn,[]) for q in ec: if o=='con': ck.out('') ck.out('Executing extra command') ck.out('') q['out']=oo q['host_os']=hos q['target_os']=tos q['device_id']=tdid r=ck.access(q) if r['return']>0: return r # Check extra files (for a given target platform) ef=d.get('distribute_extra_file',{}).get(osn,[]) r=get_paths_from_deps({'deps':deps}) if r['return']>0: return r paths=r['paths'] for q in ef: fx=q['file'] if o=='con': ck.out('') ck.out(' * Searching extra file '+fx+' ...') where=q.get('where_key','') if where=='': where='path_bin' pzz='' muoa=q.get('module_uoa','') duoa=q.get('data_uoa','') if muoa!='' and duoa!='': r=ck.access({'action':'load', 'module_uoa':muoa, 'data_uoa':duoa}) if r['return']>0: return r py=r['path'] pi=q.get('extra_dir','') if pi!='': py=os.path.join(py,pi) pzz=os.path.join(py,fx) else: for py in paths: pz=os.path.join(py,fx) if os.path.isfile(pz): pzz=pz break if pzz=='': return {'return':1, 'error':'file not found'} if o=='con': ck.out(' Found ('+pzz+') - copying ...') pz1=cus.get(pk,'') if q.get('new_file','')!='': fx=q['new_file'] pz2=os.path.join(pz1, fx) if os.path.isfile(pz2): os.remove(pz2) shutil.copyfile(pzz, pz2) # Check extra dirs (dist) in package px=os.path.join(p0,'dist') if os.path.isdir(px): if o=='con': ck.out('') ck.out('Copying extra dist directory ...') pxx=os.listdir(px) for q in pxx: q1=os.path.join(px,q) q2=os.path.join(pp,q) if o=='con': ck.out(' * '+q) if os.path.isdir(q2) or os.path.isfile(q2): shutil.rmtree(q2) shutil.copytree(q1,q2) # Prepare zip import zipfile zip_method=zipfile.ZIP_DEFLATED r=ck.list_all_files({'path':pp}) if r['return']>0: return r flx=r['list'] # Write archive if o=='con': ck.out('') ck.out('Recording generated package to '+fn+' ...') pfn=os.path.join(cur_dir,fn) try: f=open(pfn, 'wb') z=zipfile.ZipFile(f, 'w', zip_method) for fn in flx: p1=os.path.join(pp, fn) z.write(p1, 'install'+os.sep+fn, zip_method) # ck-install.json z.write(ftmp, cfg['ck_install_file_saved'], zip_method) z.close() f.close() except Exception as e: return {'return':1, 'error':'failed to prepare archive ('+format(e)+')'} if os.path.isfile(ftmp): os.remove(ftmp) return {'return':0} ############################################################################## # get all paths from deps def get_paths_from_deps(i): """ Input: { (deps) - deps (paths) - current list of paths } Output: { return - return code = 0, if successful > 0, if error (error) - error text if return > 0 } """ import os deps=i.get('deps',{}) paths=i.get('paths',[]) for k in sorted(deps, key=lambda v: deps[v].get('sort',0)): dp=deps[k] uoa=dp.get('uoa','') if uoa!='': r=ck.access({'action':'load', 'module_uoa':cfg['module_deps']['env'], 'data_uoa':uoa}) if r['return']>0: return r dp1=r['dict'] dpx=dp1.get('customize',{}) x=dpx.get('path_bin','') if x!='' and x not in paths: paths.append(x) if x=='': # Trick for some dll on Windows x=dpx.get('path_lib','') if x!='': x=os.path.dirname(x) x=os.path.join(x,'bin') if os.path.isdir(x) and x not in paths: paths.append(x) x=dpx.get('path_lib','') if x!='' and x not in paths: paths.append(x) x=dpx.get('path_include','') if x!='' and x not in paths: paths.append(x) ndpd=dp1.get('deps',{}) if len(ndpd)>0: r=get_paths_from_deps({'deps':ndpd, 'paths':paths}) if r['return']>0: return r paths=r['paths'] return {'return':0, 'paths':paths} ############################################################################## # reinstall package if already installed def reinstall(i): """ Input: { } Output: { return - return code = 0, if successful > 0, if error (error) - error text if return > 0 } """ i['reinstall']='yes' return install(i) ############################################################################## # add package with template def add(i): """ Input: { soft - specify related soft UOA (template) - if !='', use this program as template! (tags) - if !='', use these tags } Output: { return - return code = 0, if successful > 0, if error (error) - error text if return > 0 } """ o=i.get('out','') # Check related soft first suoa=i.get('soft','') if suoa=='': return {'return':1, 'error':'related software detection plugin is not specified. Specify the existing one using --soft={name from http://cKnowledge.org/shared-soft-detection-plugins.html} or create a new one using "ck add soft {repo}:soft:{name}"'} # Load soft to get UID and tags r=ck.access({'action':'load', 'module_uoa':cfg['module_deps']['soft'], 'data_uoa':suoa}) if r['return']>0: return r suid=r['data_uid'] sd=r['dict'] stags=sd.get('tags',[]) # Redirect to a universal template ... muoa=i['module_uoa'] i['original_module_uoa']=muoa i['module_uoa']=cfg['module_deps']['misc'] i['action']='prepare_entry_template' if 'cid' in i: del(i['cid']) r=ck.access(i) if r['return']>0: return r # Update newly created entry with special keys duid=r['data_uid'] duoa=r['data_uoa'] ruid=r['repo_uid'] dd=r['dict'] if 'template' in dd: del(dd['template']) if 'template_type' in dd: del(dd['template_type']) # Process tags xtags=i.get('tags','') if xtags=='': ck.out('') r=ck.inp({'text':'Enter extra version tags for your package plugin separated by comma (for example v6.0,v6): '}) xtags=r['string'].strip() tags=stags if xtags!='': for q in xtags.split(','): q=q.strip() if q not in tags: tags.append(q) dd['soft_uoa']=suid dd['tags']=tags x=sd.get('customize',{}).get('soft_file_universal','') if x!='': dd['end_full_path_universal']=x else: x=sd.get('customize',{}).get('soft_file',{}) if x!='': dd['end_full_path']=x # Update new entry ii={'action':'update', 'module_uoa':muoa, 'data_uoa':duid, 'repo_uoa':ruid, 'dict':dd, 'substitute':'yes', 'sort_keys':'yes', 'ignore_update':'yes' } if o=='con': ck.out('') ck.out('Further details about how to update meta.json and other files of your new package installation plugin:') ck.out('') ck.out(' * https://github.com/ctuning/ck/wiki/Adding-new-workflows') return ck.access(ii) ############################################################################## # internal: print community warning when package fails def print_warning(i): """ Input: { package_data_uoa package_repo_uoa package_meta } Output: { return - return code = 0, if successful > 0, if error (error) - error text if return > 0 } """ # Check if this warning is disabled in CK kernel if ck.cfg.get('skip_message_when_package_fails')!='yes': pduoa=i['package_data_uoa'] pruoa=i['package_repo_uoa'] pmeta=i['package_meta'] package_ie=pmeta.get('customize',{}).get('install_env',{}) purl=package_ie.get('PACKAGE_URL','') pc=package_ie.get('PACKAGE_GIT_CHECKOUT','') ck.out('') ck.out(' ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~') ck.out(' CK detected a PROBLEM in the third-party CK package:') # ck.out('CK packages are developed, shared and improved by the community') # ck.out('to help users automate installation and customization') # ck.out('of code and data across diverse and evolving platforms:') # ck.out('* http://cKnowledge.org/shared-packages.html') # # ck.out('') # ck.out('Therefore, they may sometimes fail with newer code versions,') # ck.out('under new settings or in previously unseen environments.') # # ck.out('') # ck.out('In such case, please help the community by fixing the problem') # ck.out('and/or reporting it via CK mailing list and related repository:') # ck.out('(please provide all details about how to reproduce it):') # # ck.out('') # ck.out('* https://groups.google.com/forum/#!forum/collective-knowledge') # # ck.out('') # ck.out('You can turn off this message as follows:') # ck.out('$ ck set kernel --var.skip_message_when_package_fails=yes') # sometimes pruoa can still be UID -> check r=ck.access({'action':'load', 'module_uoa':work['self_module_uid'], 'data_uoa':pduoa}) if r['return']==0: pduoa=r['data_uoa'] url2='' crurl='' if pduoa!='' or pruoa!='': ck.out('') ck.out(' CK package: '+pduoa) if purl!='': ck.out(' Native package URL: '+purl) if pc!='': ck.out(' Git checkout: '+pc) if pruoa!='': ck.out(' CK repo: '+pruoa) # Attempt to read info about this repo r=ck.access({'action':'load', 'module_uoa':cfg['module_deps']['repo'], 'data_uoa':pruoa}) if r['return']==0: d=r['dict'] url=d.get('url','') if url!='': url1=url+'/tree/master/package/'+pduoa url2=url+'/issues' ck.out(' CK repo URL: '+url) ck.out(' CK package URL: '+url1) ck.out(' Issues URL: '+url2) crurl=ck.cfg.get('wiki_data_web','') if crurl!='': crurl+='package/'+pduoa # ck.out('') # ck.out(' CK stable package URL: '+crurl) x1='the community' x2='https://groups.google.com/forum/#!forum/collective-knowledge' x3='https://github.com/ctuning/ck/issues' # if url2!='': # x1='the authors' # x2=url2 if crurl!='': ck.out('') ck.out(' Please, check that there is no discussion about this issue at '+crurl) ck.out('') ck.out(' Please, submit the *full* log to:') ck.out(' * '+x2) ck.out(' * '+x3) ck.out(' ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~') ck.out('') return {'return':0}