{"payload":{"feedbackUrl":"https://github.com/orgs/community/discussions/53140","repo":{"id":489109517,"defaultBranch":"main","name":"arco-era5","ownerLogin":"google-research","currentUserCanPush":false,"isFork":false,"isEmpty":false,"createdAt":"2022-05-05T19:58:23.000Z","ownerAvatar":"https://avatars.githubusercontent.com/u/43830688?v=4","public":true,"private":false,"isOrgOwned":true},"refInfo":{"name":"","listCacheKey":"v0:1725523667.0","currentOid":""},"activityList":{"items":[{"before":"d5afd90cd9aebe6a993f52e01a2d368acf68b9c2","after":null,"ref":"refs/heads/updated-automatic-script","pushedAt":"2024-09-05T08:07:47.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"dabhicusp","name":"dabhi_cusp","path":"/dabhicusp","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/123355381?s=80&v=4"}},{"before":"e9cc185ee1061842a147202a7e0cb3bbffc799bc","after":"d3d7a348c43be4a01f9f757c25b31fbbde20c05e","ref":"refs/heads/main","pushedAt":"2024-09-05T08:07:39.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"dabhicusp","name":"dabhi_cusp","path":"/dabhicusp","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/123355381?s=80&v=4"},"commit":{"message":"Updated automatic script & docker file of the arco-era5. (#85)","shortMessageHtmlLink":"Updated automatic script & docker file of the arco-era5. (#85)"}},{"before":"c2200abbba420e127c6199bef521021e373b5d26","after":"d5afd90cd9aebe6a993f52e01a2d368acf68b9c2","ref":"refs/heads/updated-automatic-script","pushedAt":"2024-09-05T08:04:14.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"dabhicusp","name":"dabhi_cusp","path":"/dabhicusp","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/123355381?s=80&v=4"},"commit":{"message":"Nit changes are done.","shortMessageHtmlLink":"Nit changes are done."}},{"before":"d97650203c1b9e8fccf56185a376a9928eff6082","after":"c2200abbba420e127c6199bef521021e373b5d26","ref":"refs/heads/updated-automatic-script","pushedAt":"2024-09-05T05:02:56.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"dabhicusp","name":"dabhi_cusp","path":"/dabhicusp","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/123355381?s=80&v=4"},"commit":{"message":"Update conda inside docker image.","shortMessageHtmlLink":"Update conda inside docker image."}},{"before":"4d61224e55fa727d23e2cf34fef5ec92e2147990","after":"d97650203c1b9e8fccf56185a376a9928eff6082","ref":"refs/heads/updated-automatic-script","pushedAt":"2024-09-04T12:10:02.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"dabhicusp","name":"dabhi_cusp","path":"/dabhicusp","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/123355381?s=80&v=4"},"commit":{"message":"import command of metview is updated.","shortMessageHtmlLink":"import command of metview is updated."}},{"before":"8fcf4223036ede08b560776325149af866c42c76","after":"4d61224e55fa727d23e2cf34fef5ec92e2147990","ref":"refs/heads/updated-automatic-script","pushedAt":"2024-09-04T09:55:46.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"dabhicusp","name":"dabhi_cusp","path":"/dabhicusp","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/123355381?s=80&v=4"},"commit":{"message":"added fsspec into the setup.py.","shortMessageHtmlLink":"added fsspec into the setup.py."}},{"before":"df7e22c729db6cfd7c3ee96839215290124d2bec","after":"8fcf4223036ede08b560776325149af866c42c76","ref":"refs/heads/updated-automatic-script","pushedAt":"2024-09-04T08:01:43.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"dabhicusp","name":"dabhi_cusp","path":"/dabhicusp","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/123355381?s=80&v=4"},"commit":{"message":"calculate the correct previous third month.","shortMessageHtmlLink":"calculate the correct previous third month."}},{"before":"62bd4d397aedd542c21ee700f48b9412f87c1c28","after":"df7e22c729db6cfd7c3ee96839215290124d2bec","ref":"refs/heads/updated-automatic-script","pushedAt":"2024-09-04T06:49:54.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"dabhicusp","name":"dabhi_cusp","path":"/dabhicusp","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/123355381?s=80&v=4"},"commit":{"message":"remove use-local-code from the raw-to-zarr-to-bq.","shortMessageHtmlLink":"remove use-local-code from the raw-to-zarr-to-bq."}},{"before":null,"after":"62bd4d397aedd542c21ee700f48b9412f87c1c28","ref":"refs/heads/updated-automatic-script","pushedAt":"2024-09-04T06:09:58.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"dabhicusp","name":"dabhi_cusp","path":"/dabhicusp","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/123355381?s=80&v=4"},"commit":{"message":"Updated automatic script of the arco-era5.","shortMessageHtmlLink":"Updated automatic script of the arco-era5."}},{"before":"f7e41171b160756deee785d075538a8b0f3f431b","after":null,"ref":"refs/heads/resize_metadata","pushedAt":"2024-08-27T14:24:09.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"dabhicusp","name":"dabhi_cusp","path":"/dabhicusp","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/123355381?s=80&v=4"}},{"before":"241e91805c56ebde2425cf8bc38bb0e2c1bd5fc6","after":"e9cc185ee1061842a147202a7e0cb3bbffc799bc","ref":"refs/heads/main","pushedAt":"2024-08-27T14:23:59.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"dabhicusp","name":"dabhi_cusp","path":"/dabhicusp","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/123355381?s=80&v=4"},"commit":{"message":"Fixed: Race condition when resizing existing datasets. (#84)\n\nI have updated all datasets by resizing to `2050-12-31`. Additionally, I\r\nadded three new attributes— `valid_time_start, valid_time_end, and\r\nlast_updated` to each Zarr store. These attributes provide essential\r\ninformation, such as the validity period of the data and the timestamp\r\nof the most recent update. These changes enhance the dataset's metadata\r\nand improve its usability, and fixed issue #81.","shortMessageHtmlLink":"Fixed: Race condition when resizing existing datasets. (#84)"}},{"before":"9539e618c13a34ee6a3da7a2c94ae7741fa6be39","after":"f7e41171b160756deee785d075538a8b0f3f431b","ref":"refs/heads/resize_metadata","pushedAt":"2024-08-27T14:20:43.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"dabhicusp","name":"dabhi_cusp","path":"/dabhicusp","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/123355381?s=80&v=4"},"commit":{"message":"Readme.md file updated.","shortMessageHtmlLink":"Readme.md file updated."}},{"before":"7ef7db86feaf94f0d8c4b58be3f939006632cea3","after":"9539e618c13a34ee6a3da7a2c94ae7741fa6be39","ref":"refs/heads/resize_metadata","pushedAt":"2024-08-27T08:03:42.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"dabhicusp","name":"dabhi_cusp","path":"/dabhicusp","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/123355381?s=80&v=4"},"commit":{"message":"readme file updated.","shortMessageHtmlLink":"readme file updated."}},{"before":null,"after":"7ef7db86feaf94f0d8c4b58be3f939006632cea3","ref":"refs/heads/resize_metadata","pushedAt":"2024-08-22T08:29:49.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"dabhicusp","name":"dabhi_cusp","path":"/dabhicusp","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/123355381?s=80&v=4"},"commit":{"message":"Fixed: Race condition when resizing datasets.","shortMessageHtmlLink":"Fixed: Race condition when resizing datasets."}},{"before":"4e2f205dbd77f21bbbf40f2ba75707756d648876","after":"241e91805c56ebde2425cf8bc38bb0e2c1bd5fc6","ref":"refs/heads/main","pushedAt":"2024-07-10T15:53:41.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"shoyer","name":"Stephan Hoyer","path":"/shoyer","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/1217238?s=80&v=4"},"commit":{"message":"Update description of data update frequency (#79)\n\nUpdates the description of data updates based on\r\nhttps://github.com/google-research/arco-era5/issues/77#issuecomment-2217505230\r\n\r\n@dabhicusp Please verify that this looks correct to you.","shortMessageHtmlLink":"Update description of data update frequency (#79)"}},{"before":"eb464582a3aaa6f93892f419dcab8cff9250734f","after":null,"ref":"refs/heads/remove_inclusive","pushedAt":"2024-07-10T12:21:47.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"dabhicusp","name":"dabhi_cusp","path":"/dabhicusp","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/123355381?s=80&v=4"}},{"before":"15673454cae04dc2650187aea429a85e5d195960","after":"4e2f205dbd77f21bbbf40f2ba75707756d648876","ref":"refs/heads/main","pushedAt":"2024-07-10T12:21:43.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"dabhicusp","name":"dabhi_cusp","path":"/dabhicusp","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/123355381?s=80&v=4"},"commit":{"message":"Remove inclusive from the source_data. (#80)\n\nRemove `inclusive=left` from the `daily_date_iterator` function of the\r\n`source_data.py` file. This will fixed the issue #78.\r\n\r\nReason: while running the script on the monthly basis this didn't\r\ninclude the last date of the month due to the `inclusive=left` so after\r\nremoving this it will added the last day's data, too.","shortMessageHtmlLink":"Remove inclusive from the source_data. (#80)"}},{"before":null,"after":"eb464582a3aaa6f93892f419dcab8cff9250734f","ref":"refs/heads/remove_inclusive","pushedAt":"2024-07-10T12:13:35.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"dabhicusp","name":"dabhi_cusp","path":"/dabhicusp","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/123355381?s=80&v=4"},"commit":{"message":"Remove inclusive from the source_data.","shortMessageHtmlLink":"Remove inclusive from the source_data."}},{"before":"946edfab8b39cd7ef56adfc32e3f4490b11749df","after":"15673454cae04dc2650187aea429a85e5d195960","ref":"refs/heads/main","pushedAt":"2024-06-25T22:24:15.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"wundersooner","name":"Rob Carver","path":"/wundersooner","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/2908070?s=80&v=4"},"commit":{"message":"Fix table of contents for README (#75)","shortMessageHtmlLink":"Fix table of contents for README (#75)"}},{"before":"276c52606aaf72a4edb3fe84c505ad1c7ec41923","after":"946edfab8b39cd7ef56adfc32e3f4490b11749df","ref":"refs/heads/main","pushedAt":"2024-06-25T22:16:26.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"wundersooner","name":"Rob Carver","path":"/wundersooner","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/2908070?s=80&v=4"},"commit":{"message":"Update README dataset description (#74)\n\nI've reorganized the README to separately introduce \"Analysis Ready\" and\r\n\"Cloud Optimized\" datasets, which an expectation that users will be most\r\ninterested in the former.\r\n\r\nI've also updated all datasets with size and chunking information,\r\ngenerating with the following snippet:\r\n```python\r\nimport xarray_beam\r\nimport math\r\n\r\ndef get_size(x):\r\n for threshold, units in [\r\n (1e6, 'MB'),\r\n (1e9, 'GB'),\r\n (1e12, 'TB'),\r\n (1e15, 'PB'),\r\n ]:\r\n if x < threshold * 1000:\r\n return x/threshold, units\r\n raise RuntimeError('unhandled size')\r\n\r\nfor path in [\r\n 'gs://gcp-public-data-arco-era5/ar/full_37-1h-0p25deg-chunk-1.zarr-v3',\r\n 'gs://gcp-public-data-arco-era5/ar/model-level-1h-0p25deg.zarr-v1',\r\n 'gs://gcp-public-data-arco-era5/co/model-level-wind.zarr-v2',\r\n 'gs://gcp-public-data-arco-era5/co/model-level-moisture.zarr-v2',\r\n 'gs://gcp-public-data-arco-era5/co/single-level-surface.zarr-v2',\r\n 'gs://gcp-public-data-arco-era5/co/single-level-reanalysis.zarr-v2',\r\n 'gs://gcp-public-data-arco-era5/co/single-level-forecast.zarr-v2', \r\n]:\r\n ds, chunks = xarray_beam.open_zarr(\r\n path, storage_options=dict(token='anon')\r\n )\r\n print()\r\n print(path)\r\n size, units = get_size(ds.sel(time=slice(\"1940\", None)).nbytes)\r\n print(f'Total size (1940-present): {size:.3g} {units}')\r\n print('Chunks:', chunks)\r\n size, units = get_size(4*math.prod(chunks.values()))\r\n print(f'Chunk size: {size:.3g} {units}')\r\n print(f'Last time: {ds.indexes[\"time\"][-1]}')\r\n```\r\n\r\nThis currently outputs:\r\n```\r\ngs://gcp-public-data-arco-era5/ar/full_37-1h-0p25deg-chunk-1.zarr-v3\r\nTotal size (1940-present): 2.05 PB\r\nChunks: {'time': 1, 'latitude': 721, 'longitude': 1440, 'level': 37}\r\nChunk size: 154 MB\r\nLast time: 2024-03-31 23:00:00\r\n\r\ngs://gcp-public-data-arco-era5/ar/model-level-1h-0p25deg.zarr-v1\r\nTotal size (1940-present): 5.88 PB\r\nChunks: {'time': 1, 'hybrid': 18, 'latitude': 721, 'longitude': 1440}\r\nChunk size: 74.8 MB\r\nLast time: 2024-03-31 23:00:00\r\n\r\ngs://gcp-public-data-arco-era5/co/model-level-wind.zarr-v2\r\nTotal size (1940-present): 664 TB\r\nChunks: {'time': 1, 'hybrid': 1, 'values': 410240}\r\nChunk size: 1.64 MB\r\nLast time: 2024-03-31 23:00:00\r\n\r\ngs://gcp-public-data-arco-era5/co/model-level-moisture.zarr-v2\r\nTotal size (1940-present): 1.54 PB\r\nChunks: {'time': 1, 'hybrid': 1, 'values': 542080}\r\nChunk size: 2.17 MB\r\nLast time: 2024-03-31 23:00:00\r\n\r\ngs://gcp-public-data-arco-era5/co/single-level-surface.zarr-v2\r\nTotal size (1940-present): 2.42 TB\r\nChunks: {'time': 1, 'values': 410240}\r\nChunk size: 1.64 MB\r\nLast time: 2024-03-31 23:00:00\r\n\r\ngs://gcp-public-data-arco-era5/co/single-level-reanalysis.zarr-v2\r\nTotal size (1940-present): 60.9 TB\r\nChunks: {'time': 1, 'values': 542080}\r\nChunk size: 2.17 MB\r\nLast time: 2024-03-31 23:00:00\r\n\r\ngs://gcp-public-data-arco-era5/co/single-level-forecast.zarr-v2\r\nTotal size (1940-present): 53.2 TB\r\nChunks: {'time': 1, 'step': 1, 'values': 542080}\r\nChunk size: 2.17 MB\r\nLast time: 2024-03-31 18:00:00\r\n```","shortMessageHtmlLink":"Update README dataset description (#74)"}},{"before":"97cdeee7a96e96544320c41b37eae6f39bd84bf7","after":"276c52606aaf72a4edb3fe84c505ad1c7ec41923","ref":"refs/heads/main","pushedAt":"2024-06-25T12:26:58.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"dabhicusp","name":"dabhi_cusp","path":"/dabhicusp","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/123355381?s=80&v=4"},"commit":{"message":"ERA5 Native vertical grid zarr(`model-level-1h-0p25deg.zarr-v1`) added. (#73)\n\nAs mentioned in the 5th phase of the roadmap, I derived a\r\nhigh-resolution version of the ERA5\r\ndataset(`gs://gcp-public-data-arco-era5/ar/model-level-1h-0p25deg.zarr-v1`).\r\nFor deriving this dataset, I regridded the datasets to lat/long grids,\r\nconverted the model level variables to pressure level variables, and\r\ncomputed some variables from the model level variables.","shortMessageHtmlLink":"ERA5 Native vertical grid zarr(model-level-1h-0p25deg.zarr-v1
) adde…"}},{"before":"45565b1f93752f404cacd56942f196b8330a87e2","after":"977101a3c6e75fa10723514d7297df4f5f0df087","ref":"refs/heads/native_vertical_grid","pushedAt":"2024-06-25T12:22:02.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"dabhicusp","name":"dabhi_cusp","path":"/dabhicusp","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/123355381?s=80&v=4"},"commit":{"message":"new code updated.","shortMessageHtmlLink":"new code updated."}},{"before":"950ad7d77126835bcb5437709d0e1908376f29bb","after":"45565b1f93752f404cacd56942f196b8330a87e2","ref":"refs/heads/native_vertical_grid","pushedAt":"2024-06-25T11:41:43.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"dabhicusp","name":"dabhi_cusp","path":"/dabhicusp","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/123355381?s=80&v=4"},"commit":{"message":"Doc string added into docker file.","shortMessageHtmlLink":"Doc string added into docker file."}},{"before":"b822f46c9b77535ed9bcca01adc0f381566124dd","after":"950ad7d77126835bcb5437709d0e1908376f29bb","ref":"refs/heads/native_vertical_grid","pushedAt":"2024-06-25T10:36:59.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"dabhicusp","name":"dabhi_cusp","path":"/dabhicusp","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/123355381?s=80&v=4"},"commit":{"message":"timestamp updated in the code.","shortMessageHtmlLink":"timestamp updated in the code."}},{"before":"dc5661583128bac2cf2297ed4c9e054c30d0240e","after":"b822f46c9b77535ed9bcca01adc0f381566124dd","ref":"refs/heads/native_vertical_grid","pushedAt":"2024-06-25T08:50:16.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"dabhicusp","name":"dabhi_cusp","path":"/dabhicusp","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/123355381?s=80&v=4"},"commit":{"message":"new constant file added.","shortMessageHtmlLink":"new constant file added."}},{"before":"7ca9c215def3df871b5689b842491f4177dfe2ab","after":null,"ref":"refs/heads/wundersooner-patch-1","pushedAt":"2024-06-25T07:03:36.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"DarshanSP19","name":"Darshan Prajapati","path":"/DarshanSP19","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/93967637?s=80&v=4"}},{"before":"3a9991abfdf989b1c6fa7becc2c09e9e22ae35a1","after":"97cdeee7a96e96544320c41b37eae6f39bd84bf7","ref":"refs/heads/main","pushedAt":"2024-06-25T07:03:32.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"DarshanSP19","name":"Darshan Prajapati","path":"/DarshanSP19","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/93967637?s=80&v=4"},"commit":{"message":"Update README.md to fix typo in roadmap (#63)","shortMessageHtmlLink":"Update README.md to fix typo in roadmap (#63)"}},{"before":"920c1a27f7e3e39373de912c5545e579e318c982","after":"dc5661583128bac2cf2297ed4c9e054c30d0240e","ref":"refs/heads/native_vertical_grid","pushedAt":"2024-06-16T15:30:16.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"dabhicusp","name":"dabhi_cusp","path":"/dabhicusp","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/123355381?s=80&v=4"},"commit":{"message":"Code refactored.","shortMessageHtmlLink":"Code refactored."}},{"before":"ab159dfb412165e9042b05458abfaf1f072503db","after":"920c1a27f7e3e39373de912c5545e579e318c982","ref":"refs/heads/native_vertical_grid","pushedAt":"2024-06-16T15:16:25.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"dabhicusp","name":"dabhi_cusp","path":"/dabhicusp","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/123355381?s=80&v=4"},"commit":{"message":"Code updated.","shortMessageHtmlLink":"Code updated."}},{"before":"72dd63fa1beee70b54bc5ec6eea3bb3682ac293c","after":"ab159dfb412165e9042b05458abfaf1f072503db","ref":"refs/heads/native_vertical_grid","pushedAt":"2024-06-14T14:17:20.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"dabhicusp","name":"dabhi_cusp","path":"/dabhicusp","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/123355381?s=80&v=4"},"commit":{"message":"Docker file and env file updated.","shortMessageHtmlLink":"Docker file and env file updated."}}],"hasNextPage":true,"hasPreviousPage":false,"activityType":"all","actor":null,"timePeriod":"all","sort":"DESC","perPage":30,"cursor":"djE6ks8AAAAErY5YuAA","startCursor":null,"endCursor":null}},"title":"Activity · google-research/arco-era5"}