test: Atomically update file data source file

.github/workflows/ci.yml

@@ -17,7 +17,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        # python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.8"]
 
     services:
       redis:
@@ -49,6 +50,28 @@ jobs:
       - name: Run tests
         run: make test
 
+      - run: make test
+      - run: make test
+      - run: make test
+      - run: make test
+      - run: make test
+      - run: make test
+      - run: make test
+      - run: make test
+      - run: make test
+      - run: make test
+      - run: make test
+      - run: make test
+      - run: make test
+      - run: make test
+      - run: make test
+      - run: make test
+      - run: make test
+      - run: make test
+      - run: make test
+      - run: make test
+      - run: make test
+
       - name: Verify typehints
         run: make lint
 
@@ -67,61 +90,61 @@ jobs:
           test_service_port: 9000
           token: ${{ secrets.GITHUB_TOKEN }}
 
-  windows:
-    runs-on: windows-latest
-
-    defaults:
-      run:
-        shell: powershell
-
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
-
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Setup DynamoDB
-        run: |
-          $ProgressPreference = "SilentlyContinue"
-          iwr -outf dynamo.zip https://s3-us-west-2.amazonaws.com/dynamodb-local/dynamodb_local_latest.zip
-          mkdir dynamo
-          Expand-Archive -Path dynamo.zip -DestinationPath dynamo
-          cd dynamo
-          cmd /c "START /b java -Djava.library.path=./DynamoDBLocal_lib -jar ./DynamoDBLocal.jar"
-
-      - name: Setup Consul
-        run: |
-          $ProgressPreference = "SilentlyContinue"
-          iwr -outf consul.zip https://releases.hashicorp.com/consul/1.4.2/consul_1.4.2_windows_amd64.zip
-          mkdir consul
-          Expand-Archive -Path consul.zip -DestinationPath consul
-          cd consul
-          sc.exe create "Consul" binPath="$(Get-Location)/consul.exe agent -dev"
-          sc.exe start "Consul"
-
-      - name: Setup Redis
-        run: |
-          $ProgressPreference = "SilentlyContinue"
-          iwr -outf redis.zip https://github.com/MicrosoftArchive/redis/releases/download/win-3.0.504/Redis-x64-3.0.504.zip
-          mkdir redis
-          Expand-Archive -Path redis.zip -DestinationPath redis
-          cd redis
-          ./redis-server --service-install
-          ./redis-server --service-start
-          Start-Sleep -s 5
-          ./redis-cli ping
-
-      - name: Install poetry
-        uses: abatilo/actions-poetry@7b6d33e44b4f08d7021a1dee3c044e9c253d6439
-
-      - name: Install requirements
-        run: poetry install --all-extras
-
-      - name: Run tests
-        run: make test
+  # windows:
+  #   runs-on: windows-latest
+  #
+  #   defaults:
+  #     run:
+  #       shell: powershell
+  #
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+  #
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - name: Set up Python ${{ matrix.python-version }}
+  #       uses: actions/setup-python@v5
+  #       with:
+  #         python-version: ${{ matrix.python-version }}
+  #
+  #     - name: Setup DynamoDB
+  #       run: |
+  #         $ProgressPreference = "SilentlyContinue"
+  #         iwr -outf dynamo.zip https://s3-us-west-2.amazonaws.com/dynamodb-local/dynamodb_local_latest.zip
+  #         mkdir dynamo
+  #         Expand-Archive -Path dynamo.zip -DestinationPath dynamo
+  #         cd dynamo
+  #         cmd /c "START /b java -Djava.library.path=./DynamoDBLocal_lib -jar ./DynamoDBLocal.jar"
+  #
+  #     - name: Setup Consul
+  #       run: |
+  #         $ProgressPreference = "SilentlyContinue"
+  #         iwr -outf consul.zip https://releases.hashicorp.com/consul/1.4.2/consul_1.4.2_windows_amd64.zip
+  #         mkdir consul
+  #         Expand-Archive -Path consul.zip -DestinationPath consul
+  #         cd consul
+  #         sc.exe create "Consul" binPath="$(Get-Location)/consul.exe agent -dev"
+  #         sc.exe start "Consul"
+  #
+  #     - name: Setup Redis
+  #       run: |
+  #         $ProgressPreference = "SilentlyContinue"
+  #         iwr -outf redis.zip https://github.com/MicrosoftArchive/redis/releases/download/win-3.0.504/Redis-x64-3.0.504.zip
+  #         mkdir redis
+  #         Expand-Archive -Path redis.zip -DestinationPath redis
+  #         cd redis
+  #         ./redis-server --service-install
+  #         ./redis-server --service-start
+  #         Start-Sleep -s 5
+  #         ./redis-cli ping
+  #
+  #     - name: Install poetry
+  #       uses: abatilo/actions-poetry@7b6d33e44b4f08d7021a1dee3c044e9c253d6439
+  #
+  #     - name: Install requirements
+  #       run: poetry install --all-extras
+  #
+  #     - name: Run tests
+  #       run: make test

Makefile

@@ -28,7 +28,7 @@ install:
 .PHONY: test
 test: #! Run unit tests
 test: install
-	@poetry run pytest $(PYTEST_FLAGS)
+	@poetry run pytest $(PYTEST_FLAGS) ldclient/testing/test_file_data_source.py
 
 .PHONY: lint
 lint: #! Run type analysis and linting checks

ldclient/impl/integrations/files/file_data_source.py

@@ -84,7 +84,10 @@ def _load_all(self):
         for path in self._paths:
             try:
                 self._load_file(path, all_data)
-            except Exception as e:
+            except FileDataSourceEmpty:
+                log.warning('No flag data found in any file, continuing with current state')
+                return
+            except (Exception, FileNotFoundError) as e:
                 log.error('Unable to load flag data from "%s": %s' % (path, repr(e)))
                 traceback.print_exc()
                 if self._data_source_update_sink is not None:
@@ -112,6 +115,10 @@ def _load_file(self, path, all_data):
         with open(path, 'r') as f:
             content = f.read()
         parsed = self._parse_content(content)
+
+        if parsed is None:
+            raise FileDataSourceEmpty()
+
         for key, flag in parsed.get('flags', {}).items():
             _sanitize_json_item(flag)
             self._add_item(all_data, FEATURES, flag)
@@ -165,6 +172,13 @@ def __init__(self, resolved_paths, reloader):
 
             class LDWatchdogHandler(watchdog.events.FileSystemEventHandler):
                 def on_any_event(self, event):
+                    if isinstance(event, watchdog.events.FileDeletedEvent):
+                        return
+
+                    if isinstance(event, watchdog.events.FileMovedEvent) and event.dest_path in watched_files:
+                        reloader()
+                        return
+
                     if event.src_path in watched_files:
                         reloader()
 
@@ -213,4 +227,9 @@ def _check_file_times(self):
                     ret[path] = os.path.getmtime(path)
                 except:
                     ret[path] = None
+
             return ret
+
+
+class FileDataSourceEmpty(Exception):
+    pass

ldclient/testing/test_file_data_source.py

@@ -2,6 +2,8 @@
 import os
 from typing import List
 
+from ldclient.impl.util import log
+
 import pytest
 import tempfile
 import threading
@@ -116,10 +118,14 @@ def make_temp_file(content):
     os.close(f)
     return path
 
-def replace_file(path, content):
+def update_file(path, content):
     with open(path, 'w') as f:
         f.write(content)
 
+def replace_file(path, content):
+    new_file = make_temp_file(content)
+    os.replace(new_file, path)
+
 def test_does_not_load_data_prior_to_start():
     path = make_temp_file('{"flagValues":{"key":"value"}}')
     try:
@@ -221,7 +227,7 @@ def test_does_not_allow_duplicate_keys():
         os.remove(path1)
         os.remove(path2)
 
-def test_does_not_reload_modified_file_if_auto_update_is_off():
+def test_does_not_reload_modified_file_if_auto_update_is_off_when_replacing_file():
     path = make_temp_file(flag_only_json)
     try:
         source = make_data_source(Config("SDK_KEY"), paths = path)
@@ -234,15 +240,29 @@ def test_does_not_reload_modified_file_if_auto_update_is_off():
     finally:
         os.remove(path)
 
-def do_auto_update_test(options):
+def test_does_not_reload_modified_file_if_auto_update_is_off_when_updating_file():
     path = make_temp_file(flag_only_json)
+    try:
+        source = make_data_source(Config("SDK_KEY"), paths = path)
+        source.start()
+        assert len(store.all(SEGMENTS, lambda x: x)) == 0
+        time.sleep(0.5)
+        update_file(path, segment_only_json)
+        time.sleep(0.5)
+        assert len(store.all(SEGMENTS, lambda x: x)) == 0
+    finally:
+        os.remove(path)
+
+def do_auto_update_test(options, update_fn):
+    path = make_temp_file(flag_only_json)
+
     options['paths'] = path
     try:
         source = make_data_source(Config("SDK_KEY"), **options)
         source.start()
         assert len(store.all(SEGMENTS, lambda x: x)) == 0
         time.sleep(0.5)
-        replace_file(path, segment_only_json)
+        update_fn(path, segment_only_json)
         deadline = time.time() + 20
         while time.time() < deadline:
             time.sleep(0.1)
@@ -252,11 +272,17 @@ def do_auto_update_test(options):
     finally:
         os.remove(path)
 
-def test_reloads_modified_file_if_auto_update_is_on():
-    do_auto_update_test({ 'auto_update': True })
+def test_reloads_modified_file_if_auto_update_is_on_when_replacing_file():
+    do_auto_update_test({ 'auto_update': True }, replace_file)
+
+def test_reloads_modified_file_in_polling_mode_when_replacing_file():
+    do_auto_update_test({ 'auto_update': True, 'force_polling': True, 'poll_interval': 0.1 }, replace_file)
+
+def test_reloads_modified_file_if_auto_update_is_on_when_updating_file():
+    do_auto_update_test({ 'auto_update': True }, update_file)
 
-def test_reloads_modified_file_in_polling_mode():
-    do_auto_update_test({ 'auto_update': True, 'force_polling': True, 'poll_interval': 0.1 })
+def test_reloads_modified_file_in_polling_mode_when_updating_file():
+    do_auto_update_test({ 'auto_update': True, 'force_polling': True, 'poll_interval': 0.1 }, update_file)
 
 def test_evaluates_full_flag_with_client_as_expected():
     path = make_temp_file(all_properties_json)