all views for this version done, added to storm

src/analytics.py

@@ -113,7 +113,7 @@ def gen_v_playlist_info(self, playlist_ids=[]):
             df.loc[playlist_id, "description"] = playlist_data["info"]["description"]
             df.loc[playlist_id, "last_collected"] = playlist_data["last_collected"]
 
-        df.index.rename("playlist_id", inplace=True)
+        df.index.rename("playlist", inplace=True)
         return df.reset_index()
 
     # Run Views
@@ -136,6 +136,7 @@ def gen_v_run_history(self, storm_names=[]):
             for run in self.tqdm(runs):
 
                 # Copying
+                run_df.loc[run["_id"], 'storm_name'] = storm
                 run_df.loc[run['_id'], 'run_date'] = run['run_date']
                 run_df.loc[run['_id'], 'start_date'] = run['start_date']
 
@@ -161,7 +162,19 @@ def gen_v_run_history(self, storm_names=[]):
         df.index.rename('run_id', inplace=True)
         return df.reset_index()
 
+    def gen_v_track_info(self, tracks=[]):
+        """
+        Essentially a copy and paste of the tracks in the DB
+        """
+
+        if len(tracks) == 0:
+            self.print("No tracks supplied, running it for all.")
+            tracks = self.sdb.get_tracks()
+
+        df = pd.DataFrame(self.sdb.get_track_info(tracks))
+        df.rename(columns={'_id':'track'})
 
+        return df
 
 class StormAnalyticsController:
     """
@@ -181,7 +194,8 @@ def __init__(self, verbocity=3):
         self.view_map = {'single_playlist_history':self.sag.gen_v_single_playlist_history,
                              'playlist_history':self.sag.gen_v_playlist_history,                                                 
                              'playlist_info':self.sag.gen_v_playlist_info,
-                             'run_history':self.sag.gen_v_run_history}
+                             'run_history':self.sag.gen_v_run_history,
+                             'track_info':self.sag.gen_v_track_info}
 
         # Verbocity
         self.print = print if verbocity > 0 else lambda x: None
@@ -205,7 +219,8 @@ def analytics_pipeline(self, custom_pipeline=None):
             # SDB -> SADB
             pipeline['view_generation_pipeline'] = [('playlist_history', {"playlist_ids":[]}),
                                                     ('playlist_info', {"playlist_ids":[]}),
-                                                    ('run_history', {"storm_names":[]})]
+                                                    ('run_history', {"storm_names":[]}),
+                                                    ('track_info', {"tracks":[]})]
 
         else:
             pipeline = custom_pipeline

src/db.py

@@ -388,6 +388,36 @@ def get_track_artists(self, track):
             return [] # not good, for downstream bug fixing
             raise ValueError(f"Track {track} not found or doesn't have any artists.")
 
+    def get_tracks(self):
+        """
+        Returns a list of all tracks in the database.
+        """
+        q = {}
+        cols = {"_id":1}
+        r = list(self.tracks.find(q, cols))
+
+        return [x["_id"] for x in r]
+
+    def get_track_info(self, track_ids):
+        """
+        Returns all available information for every track in track_ids.
+        Done in batches as it is a large database.
+        """
+
+        # Check if needs to be done in batches
+        id_lim = 50000
+        batches = np.array_split(track_ids, int(np.ceil(len(track_ids)/id_lim)))
+        result = []
+        for batch in tqdm(batches):
+
+            q = {"_id":{"$in":batch.tolist()}}
+            cols = {"artists":0, "audio_analysis":0}
+            r = list(self.tracks.find(q, cols))
+            result.extend(r)
+
+        return result
+
+
     # Track Write Endpoints
     def update_tracks(self, track_info_list):
         """

src/storm.py

@@ -21,6 +21,7 @@
 from .db import *
 from .storm_client import *
 from .runner import *
+from .analytics import *
 
 class Storm:
     """
@@ -30,6 +31,7 @@ def __init__(self, storm_names, start_date=None):
 
         self.print_initial_screen()
         self.storm_names = storm_names
+        self.sac = StormAnalyticsController()
 
     def print_initial_screen(self):
 
@@ -42,4 +44,8 @@ def Run(self):
         for storm_name in self.storm_names:
             StormRunner(storm_name).Run()
 
+        print("Done Runnings, rebuilding storm_analytics")
+        self.sac.analytics_pipeline()
+
+
 #Storm(['film_vg_instrumental', 'contemporary_lyrical']).Run()