# # Collective Knowledge (demonstrating MILEPOST technology powered by Collective Knowledge Framework) # # See CK LICENSE.txt for licensing details # See CK COPYRIGHT.txt for copyright details # # Developer: Grigori Fursin, Grigori.Fursin@cTuning.org, http://fursin.net # cfg={} # Will be updated by CK (meta description of this module) work={} # Will be updated by CK (temporal data) ck=None # Will be updated by CK (initialized CK kernel) # Local settings hextra='
\n' hextra+='This is a community-driven R&D: \n' hextra+=' [ CK wiki , \n' hextra+=' MILEPOST GCC wiki , \n' hextra+=' CK-MILEPOST GitHub , \n' hextra+=' vision papers (cTuning , \n' hextra+=' MILEPOST GCC , \n' hextra+=' crowd-tuning, \n' hextra+=' CK), \n' hextra+='YouTube intro ]
\n' hextra+='
Optimizations results are continuously shared by volunteers across diverse programs, data sets and platforms (also contributed by the community in the CK format): ' hextra+='GCC , \n' hextra+='LLVM \n' hextra+='
\n' default_prog_uoa='sample-milepost-codelet' prog_kernel_c='kernel.c' ici1='ici_features_function.main.' ici2='ici_passes_function.main.txt' file_features='features.P' file_features_out='features.FT' ############################################################################## # Initialize module def init(i): """ Input: {} Output: { return - return code = 0, if successful > 0, if error (error) - error text if return > 0 } """ return {'return':0} ############################################################################## # show dashboard def show(i): """ Input: { } Output: { return - return code = 0, if successful > 0, if error (error) - error text if return > 0 (predicted_opt) - MILEPOST prediction if features are provided and one configuration identified } """ import os import shutil import time import copy form_name='milepost_web_form' sh=i.get('skip_html','') # State: extract ae=False if 'action_extract' in i: ae=True # State reset ar=False if 'action_reset' in i: ar=True ae=False # Start HTML h='' h+='
\n' h+='\n\n\n\n' # h+='

Aggregated results from Caffe crowd-benchmarking (time, accuracy, energy, cost, ...)

\n' # import json # h+=json.dumps(i,sort_keys=True) h+=hextra st='' if i.get('widget','')=='yes': url0=i.get('prepared_url0','') url1=i.get('prepared_url1','') form_name=i.get('prepared_form_name','') else: # Check host URL prefix and default module/action rx=ck.access({'action':'form_url_prefix', 'module_uoa':'wfe', 'host':i.get('host',''), 'port':i.get('port',''), 'template':i.get('template','')}) if rx['return']>0: return rx url0=rx['url'] template=rx['template'] url=url0 action=i.get('action','') muoa=i.get('module_uoa','') url+='action=index&module_uoa=wfe&native_action='+action+'&'+'native_module_uoa='+muoa url1=url # Start form r=ck.access({'action':'start_form', 'module_uoa':cfg['module_deps']['wfe'], 'url':url1, 'name':form_name}) if r['return']>0: return r h+=r['html'] url=url0 onchange='document.'+form_name+'.submit();' # Header h+='

Predict optimization using MILEPOST program features via CK (CK JSON API demo)

\n' # First params h+='\n' h+=' \n' h+=' \n' x='' # Get default program (to compare or reset) r=ck.access({'action':'load', 'module_uoa':cfg['module_deps']['program'], 'data_uoa':default_prog_uoa}) if r['return']>0: return r dprog=r['dict'] p=r['path'] prog_path=r['path'] default_prog_uid=r['data_uid'] fn=os.path.join(p,prog_kernel_c) r=ck.load_text_file({'text_file':fn}) if r['return']>0: return r default_prog=r['string'].replace('\\r','').strip() prog=i.get('program_sources','').replace('\\r','').strip() # Strange size, so save file and load it again rx=ck.gen_tmp_file({'prefix':'tmp-', 'suffix':'.tmp'}) if rx['return']>0: return rx ftmp=rx['file_name'] r=ck.save_text_file({'text_file':ftmp, 'string':prog}) if r['return']>0: return r r=ck.load_text_file({'text_file':ftmp, 'delete_after_read':'yes'}) if r['return']>0: return r prog=r['string'] if prog=='' or ar: prog=default_prog y='width:600px;height:270px;' x='\n' h+=' \n' h+=' \n' h+=' \n' x='' dt=[{'name':'C', 'value':'CC'}, {'name':'C++','value':'CXX'}, {'name':'Fortran', 'value':'FC'}, ] plang=i.get('program_language','') if plang=='' or ar: plang='CC' ii={'action':'create_selector', 'module_uoa':cfg['module_deps']['wfe'], 'data':dt, 'name':'program_language', 'onchange':onchange, 'skip_sort':'yes', 'selected_value':plang} r=ck.access(ii) if r['return']>0: return r x=r['html'] h+=' \n' h+=' \n' h+=' \n' x='' flags=i.get('milepost_flags','') if flags=='' or ar: flags='-O1' flags=safe_str(flags) x='\n' h+=' \n' h+=' \n' h+=' \n' x='' mpass=i.get('milepost_pass','') if mpass=='' or ar: mpass='fre' mpass=safe_str(mpass) x='\n' h+=' \n' h+=' \n' h+='
Program source code'+x+'
Program language'+x+'
MILEPOST GCC flags'+x+'
MILEPOST GCC pass to extract features'+x+'
\n' # Buttons h+='

\n' h+='\n' h+='\n' h+='
\n' # Check features from cmd features={} if not ar: for k in i: if k.startswith('mft'): features[k[1:]]=i[k] # If extract if ae: if prog!=default_prog: # Create dummy copy dprog['compiler_env']='CK_'+plang ii={'action':'add', 'module_uoa':cfg['module_deps']['program'], 'dict':dprog} rlru=ck.cfg.get('record_local_repo_uoa','') if rlru!='': ii['repo_uoa']=rlru r=ck.access(ii) if r['return']>0: return r prog_uid=r['data_uid'] prog_path=r['path'] fn=os.path.join(prog_path,prog_kernel_c) r=ck.save_text_file({'text_file':fn, 'string':prog}) if r['return']>0: return r else: prog_uid=default_prog_uid # Remove tmp dir px=os.path.join(prog_path,'tmp') if os.path.isdir(px): shutil.rmtree(px) time.sleep(1) # Compile r=ck.access({'action':'compile', 'module_uoa':cfg['module_deps']['program'], 'data_uoa':prog_uid, 'flags':'--ct-extract-features '+flags, 'env':{'ICI_PROG_FEAT_PASS':mpass}}) if r['return']>0: h+='

Compilation failed: '+r['error']+'
\n' prog_uid='' else: r=ck.access({'action':'load', 'module_uoa':cfg['module_deps']['program'], 'data_uoa':prog_uid}) if r['return']>0: return r prog_path=r['path'] f1=os.path.join(prog_path,'tmp',ici1+mpass+'.ft') f2=os.path.join(prog_path,'tmp',ici2) if not os.path.isfile(f1) or not os.path.isfile(f2): h+='

WARNING: features were not extracted - file with features is not found!\n' else: r=ck.load_text_file({'text_file':f1}) if r['return']>0: return r s1=r['string'].strip() r=ck.load_text_file({'text_file':f2}) if r['return']>0: return r s2=r['string'].strip() h+='


Passes during feature extration:
\n' h+='

'+s2+'
' # Parse features fx=s1.split(',') for q in fx: qq=q.strip().split('=') if len(qq)==2: k=qq[0][2:] v=float(qq[1]) features['ft'+str(k)]=v h+='

\n' # Show features r=ck.access({'action':'load', 'module_uoa':cfg['module_deps']['module'], 'data_uoa':cfg['module_deps']['program.static.features']}) if r['return']>0: return r d=r['dict'] ml1=d.get('milepost_description_ctuning_page','') ml2=d.get('milepost_features_description',{}) ml3=d.get('milepost_normalization_feature','') h+='

MILEPOST features (main function)

\n' h+='\n' h+=' \n' for k in sorted(ml2, key=lambda x: int(x)): q=ml2[k] desc=q.get('desc','') ft='ft'+str(k) h+=' \n' h+='
FeatureDescription
'+ft+' ('+desc+')
\n' # Buttons h+='

\n' h+='\n' h+='
\n' # Find similar # h+='

\n' # h+='\n' # h+='\n' # h+='
\n' # If selected solution smuoa='' sduoa='' for k in i: if k.startswith('view_solution_'): kk=k[14:] j=kk.find('_') if j>0: smuoa=kk[:j] sduoa=kk[j+1:] if smuoa=='' and sduoa=='': scenario=i.get('scenario','') if scenario=='': scenario='8289e0cf24346aa7' i['scenario']=scenario ii={} if not ar: ii=copy.deepcopy(i) ii['action']='show' ii['module_uoa']='program.optimization' ii['widget']='yes' ii['prepared_url0']=url0 ii['prepared_url1']=url1 ii['prepared_form_name']=form_name ii['prepared_scenario_tags']='program-features' r=ck.access(ii) if r['return']>0: return r results=r.get('results',[]) h+='


'+r.get('html','') st+=r.get('style','') else: results=[{'module_uoa':smuoa, 'data_uoa':sduoa}] # Save a few vars to keep state popt='' if len(results)==1: # showing unique result ***************************************************** rr=results[0] muoa=rr['module_uoa'] duoa=rr['data_uoa'] ii={'action':'html_viewer', 'module_uoa':muoa, 'data_uoa':duoa, 'url_base':url0, 'features':features} r=ck.access(ii) if r['return']>0: return r popt=r.get('predicted_opt','') h+='

'+r.get('html','') st+=r.get('style','') else: h+='

\n' if sh=='yes': h='' st='' return {'return':0, 'html':h, 'style':st, 'predicted_opt':popt} ############################################################################## # make safe strings for CMD ... def safe_str(s): s=s.replace(';',' ').replace('&',' ').replace('>',' ').replace('<',' ') return s ############################################################################## # open dashboard def dashboard(i): """ Input: { } Output: { return - return code = 0, if successful > 0, if error (error) - error text if return > 0 } """ i['action']='browser' i['cid']='' i['module_uoa']='' i['extra_url']='native_action=show&native_module_uoa=milepost' return ck.access(i) ############################################################################## # access MILEPOST AI via unified CK JSON API def ask_ai(i): """ Input: { } Output: { return - return code = 0, if successful > 0, if error (error) - error text if return > 0 } """ return {'return':0} ############################################################################## # access XSB remotely on devices with constraint memory def remote_xsb(i): """ Input: { (input) - XSB input file } Output: { return - return code = 0, if successful > 0, if error (error) - error text if return > 0 } """ # Setting output o=i.get('out','') er=i.get('exchange_repo','') if er=='': er=ck.cfg['default_exchange_repo_uoa'] esr=i.get('exchange_subrepo','') if esr=='': esr=ck.cfg['default_exchange_subrepo_uoa'] if i.get('local','')=='yes': er='local' esr='' # Load featlstn.P r=ck.load_text_file({'text_file':file_features}) if r['return']>0: return r fx=r['string'] r=ck.access({'action':'remote_xsb_api', 'repo_uoa':er, 'remote_repo_uoa':esr, 'module_uoa':work['self_module_uid'], 'input':fx}) if r['return']>0: return r # Save output ft=r['output'] r=ck.save_text_file({'text_file':file_features_out, 'string':ft}) if r['return']>0: return r return {'return':0} ############################################################################## # XSB remote CK API def remote_xsb_api(i): """ Input: { input - input source } Output: { return - return code = 0, if successful > 0, if error (error) - error text if return > 0 output - output features } """ import os import tempfile import shutil import time fi=i['input'] fo='features.FT' cur_dir=os.getcwd() # Go to tmp dir tdx=tempfile.gettempdir() rx=ck.gen_tmp_file({'prefix':'tmp-ck-xsb-'}) if rx['return']>0: return rx ftmp=rx['file_name'] td=os.path.join(tdx,ftmp) if not os.path.isdir(td): os.makedirs(td) os.chdir(td) # Save file r=ck.save_text_file({'text_file':file_features, 'string':fi}) if r['return']>0: return r # Resolve deps on ctuning-cc-plugins r=ck.access({'action':'set', 'module_uoa':cfg['module_deps']['env'], 'tags':'plugin,milepost,ctuning'}) if r['return']>0: return r bat=r['bat'] prog=r['dict']['customize']['full_path'] ss=bat+'\n'+prog os.system(ss) r=ck.load_text_file({'text_file':file_features_out, 'delete_after_read':'yes'}) if r['return']>0: return r ft=r['string'] if os.path.isdir(td): shutil.rmtree(td) time.sleep(1) return {'return':0, 'output':ft} ############################################################################## # CK AI JSON API for web (needed to automatically find such function from higher-level CK AI API) def ask_ai_web(i): """ Input: { } Output: { return - return code = 0, if successful > 0, if error (error) - error text if return > 0 } """ return show(i) ############################################################################## # return json instead of html in show (needed for CK AI API) def show_json(i): """ Input: { } Output: { return - return code = 0, if successful > 0, if error (error) - error text if return > 0 } """ r=show(i) if 'html' in r: del(r['html']) if 'style' in r: del(r['style']) return r