Add release date to scraping

ddxv · Oct 17, 2023 · e90245b · e90245b
1 parent 7ff0f34
commit e90245b
Show file tree

Hide file tree

Showing 4 changed files with 9 additions and 1 deletion.
diff --git a/adscrawler/app_stores/apple.py b/adscrawler/app_stores/apple.py
@@ -105,7 +105,6 @@ def scrape_app_ios(store_id: str, country: str) -> dict:
     # NOTE: averageUserRating, Rating_count, Histogram are country specific
     scraper = AppStoreScraper()
     result: dict = scraper.get_app_details(store_id, country=country, add_ratings=True)
-
     return result
 
 
@@ -119,6 +118,7 @@ def clean_ios_app_df(df: pd.DataFrame) -> pd.DataFrame:
             "minimum_OsVersion": "minimum_android",
             "primaryGenreName": "category",
             "bundleId": "bundle_id",
+            "releaseDate": "release_date",
             "currentVersionReleaseDate": "store_last_updated",
             "artistId": "developer_id",
             "artistName": "developer_name",
@@ -150,6 +150,9 @@ def clean_ios_app_df(df: pd.DataFrame) -> pd.DataFrame:
         store_last_updated=pd.to_datetime(df["store_last_updated"]).dt.strftime(
             "%Y-%m-%d %H:%M"
         ),
+        release_date=pd.to_datetime(
+            df["release_date"], format="%Y-%m-%dT%H:%M:%SZ"
+        ).dt.date,
     )
     try:
         df["histogram"] = df["user_ratings"].apply(

diff --git a/adscrawler/app_stores/google.py b/adscrawler/app_stores/google.py
@@ -42,6 +42,8 @@ def clean_google_play_app_df(df: pd.DataFrame) -> pd.DataFrame:
             "updated": "store_last_updated",
             "reviews": "review_count",
             "ratings": "rating_count",
+            "summary": "short_description",
+            "released": "release_date",
             "containsAds": "ad_supported",
             "offersIAP": "in_app_purchases",
             "url": "store_url",
@@ -64,6 +66,7 @@ def clean_google_play_app_df(df: pd.DataFrame) -> pd.DataFrame:
         store_last_updated=pd.to_datetime(
             df["store_last_updated"], unit="s"
         ).dt.strftime("%Y-%m-%d %H:%M"),
+        release_date=pd.to_datetime(df["release_date"], format="%b %d, %Y").dt.date,
     )
     return df
 

diff --git a/adscrawler/app_stores/scrape_stores.py b/adscrawler/app_stores/scrape_stores.py
@@ -584,6 +584,7 @@ def log_crawl_results(app_df: pd.DataFrame, database_connection: PostgresCon) ->
     "size",
     "minimum_android",
     "review_count",
+    "release_date",
     "content_rating",
     "store_last_updated",
     "developer_email",

diff --git a/pg-ddl/db.sql b/pg-ddl/db.sql
@@ -553,6 +553,7 @@ CREATE TABLE public.store_apps (
 	in_app_purchases bool NULL,
 	editors_choice bool NULL,
     icon_url_512 varchar NULL,
+    release_date date NULL,
 	created_at timestamp NULL DEFAULT timezone('utc'::text, now()),
 	updated_at timestamp NULL DEFAULT timezone('utc'::text, now()),
 	crawl_result int4 NULL,