Add merging data

CHANGELOG

@@ -6,10 +6,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
-## [0.0.8] - 2020-07-16
+## [0.0.8] - 2020-07-17
 ### Added
 - memory optimizations
 - Add check of molecule existance in .ndx file
+- Merge data into unified dataset ready to be used
 
 ### Changed
 - Update I/O to functions

main.py

 import os
+import pandas as pd
 import tooba_f as tbf
 import tooba_gmx as tbgmx
 ###################################################
@@ -9,7 +10,7 @@ import tooba_gmx as tbgmx
 ###################################################
 SYSTEM_NAME='test'
 DISCET=[3.5, 3.5, 3.5]
-NUM_FR=100
+NUM_FR=2
 TRAJ=SYSTEM_NAME+'/eq_traj.trr'
 GRO=SYSTEM_NAME+'/eq_final.gro'
 TPR=SYSTEM_NAME+'/eq_run.tpr'
@@ -112,6 +113,7 @@ for i in GROUPS.keys():
         prev=str(j)
 ###################################################
 #--------------------------------------------------
+mrg_data={}
 for i in GROUPS.keys():
 #not COMBINE section
     if i!='COMBINE': 
@@ -186,9 +188,9 @@ for i in GROUPS.keys():
                     uniq_id=tbf.frompickle(sv_index[j]['name'])
 #--------------------------------------------------
                 if j=='density' and sv_index[j]['status']=='not exist':
-                    dens_df={}
+                    dens_dict={}
                     for iidd in uniq_id.keys():
-                        dens_df[iidd]={}
+                        dens_dict[iidd]={}
                         fl='./'+uniq_id[iidd]['system']+'/gmx_ndx/'+uniq_id[iidd]['ndx_file']                  
                         cnt=-1
                         for mol in locals()[i].keys():
@@ -205,16 +207,19 @@ for i in GROUPS.keys():
                                     if len(tmp)<len(peaks):
                                         peaks=tmp
                                     tmp=peaks
-
-                            dens_df[iidd][mol]=peaks
-                        sv_data=dens_df
+                            dens_nm=mol+'_dens'
+                            dens_dict[iidd][dens_nm]=peaks
+                        sv_data=dens_dict
+                        mrg_data[j]=[dens_dict,[]]
+                    del dens_dict
                 elif j=='density' and sv_index[j]['status']=='exist':
-                    dens_df=tbf.frompickle(sv_index[j]['name'])
+                    dens_dict=tbf.frompickle(sv_index[j]['name'])
+                    mrg_data[j]=[dens_dict,[]]
+                    del dens_dict
 #--------------------------------------------------
                 if j=='gmx_ndx':
                     box_res=tbf.frompickle('./'+SYSTEM_NAME+'/'+SYSTEM_NAME+'_'+i+'_box.pkl')
                     tbf.togmxndx(box_res, fld='./'+SYSTEM_NAME, sv_name=SYSTEM_NAME+'_'+i)
-                    del box_res
 #--------------------------------------------------
             # Save module
             if len(j)==3:
@@ -271,8 +276,12 @@ for i in GROUPS.keys():
                         except TypeError: #in case of one frame
                             tot_avg[hsh]=sum([avg[sub]])/len([avg[sub]])
                     sv_data=tot_avg
+                    mrg_data[j[1]]=[tot_avg,['Tilt[degrees]']]
+                    del tot_avg
                 elif j[1]=='tilt' and sv_index[str(j)]['status']=='exist':
                     tot_avg=tbf.frompickle(sv_index[str(j)]['name'])
+                    mrg_data[j[1]]=[tot_avg,['Tilt[degrees]']]
+                    del tot_avg
 #--------------------------------------------------
             # Save module
             if len(j)==3:
@@ -288,4 +297,21 @@ for i in GROUPS.keys():
                             if k=='json':    
                                 tbf.tojson(fl=sv_data, sv_name='./'+SYSTEM_NAME+'/'+SYSTEM_NAME+'_'+i+'_'+j[2])             
 ###################################################
+#Merge data
+tbf.topickle(fl=mrg_data, sv_name='./'+SYSTEM_NAME+'/'+SYSTEM_NAME+'_merge')
+print(' ')
+print('Merging data of:')
+print('==============================')
+for tp in mrg_data.keys():
+    if len(mrg_data[tp][1])!=0:
+        df=pd.DataFrame.from_dict(mrg_data[tp][0], orient='index',columns=mrg_data[tp][1])
+    else:
+        df=pd.DataFrame.from_dict(mrg_data[tp][0], orient='index')
+    try:
+        data_df=data_df.join(df)
+    except:
+        data_df=df.copy()
+        continue
+tbf.topickle(fl=data_df, sv_name='./'+SYSTEM_NAME+'/'+SYSTEM_NAME+'_dataset')
+###################################################
 ###################################################
\ No newline at end of file

requirements.txt

@@ -3,3 +3,9 @@ scipy
 mpl_toolkits
 matplotlib
 pytrr
+pandas
+pickle
+json
+progress.bar
+subprocess
+hashlib
\ No newline at end of file

tooba_f.py

@@ -6,6 +6,7 @@ import scipy.optimize
 import pickle as pkl
 import json
 import re
+import pandas as pd
 from progress.bar import Bar
 
 from pytrr import (
@@ -764,3 +765,6 @@ def togmxndx(box_res, fld, sv_name):
 
         bar.next()
     bar.finish()
+
+def dict2pd(d, col=[]):
+    return pd.DataFrame.from_dict(d, orient='index', columns=col)
\ No newline at end of file