test: Atomically update file data source file

.github/workflows/ci.yml

@@ -17,7 +17,8 @@ jobs:
  strategy:
  fail-fast: false
  matrix:
- python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+ # python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+ python-version: ["3.8"]
 
  services:
  redis:
@@ -49,6 +50,28 @@ jobs:
  - name: Run tests
  run: make test
 
+ - run: make test
+ - run: make test
+ - run: make test
+ - run: make test
+ - run: make test
+ - run: make test
+ - run: make test
+ - run: make test
+ - run: make test
+ - run: make test
+ - run: make test
+ - run: make test
+ - run: make test
+ - run: make test
+ - run: make test
+ - run: make test
+ - run: make test
+ - run: make test
+ - run: make test
+ - run: make test
+ - run: make test
+
  - name: Verify typehints
  run: make lint
 
@@ -67,61 +90,61 @@ jobs:
  test_service_port: 9000
  token: ${{ secrets.GITHUB_TOKEN }}
 
- windows:
- runs-on: windows-latest
-
- defaults:
- run:
- shell: powershell
-
- strategy:
- fail-fast: false
- matrix:
- python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
-
- steps:
- - uses: actions/checkout@v4
- - name: Set up Python ${{ matrix.python-version }}
- uses: actions/setup-python@v5
- with:
- python-version: ${{ matrix.python-version }}
-
- - name: Setup DynamoDB
- run: |
- $ProgressPreference = "SilentlyContinue"
- iwr -outf dynamo.zip https://s3-us-west-2.amazonaws.com/dynamodb-local/dynamodb_local_latest.zip
- mkdir dynamo
- Expand-Archive -Path dynamo.zip -DestinationPath dynamo
- cd dynamo
- cmd /c "START /b java -Djava.library.path=./DynamoDBLocal_lib -jar ./DynamoDBLocal.jar"
-
- - name: Setup Consul
- run: |
- $ProgressPreference = "SilentlyContinue"
- iwr -outf consul.zip https://releases.hashicorp.com/consul/1.4.2/consul_1.4.2_windows_amd64.zip
- mkdir consul
- Expand-Archive -Path consul.zip -DestinationPath consul
- cd consul
- sc.exe create "Consul" binPath="$(Get-Location)/consul.exe agent -dev"
- sc.exe start "Consul"
-
- - name: Setup Redis
- run: |
- $ProgressPreference = "SilentlyContinue"
- iwr -outf redis.zip https://github.com/MicrosoftArchive/redis/releases/download/win-3.0.504/Redis-x64-3.0.504.zip
- mkdir redis
- Expand-Archive -Path redis.zip -DestinationPath redis
- cd redis
- ./redis-server --service-install
- ./redis-server --service-start
- Start-Sleep -s 5
- ./redis-cli ping
-
- - name: Install poetry
- uses: abatilo/actions-poetry@7b6d33e44b4f08d7021a1dee3c044e9c253d6439
-
- - name: Install requirements
- run: poetry install --all-extras
-
- - name: Run tests
- run: make test
+ # windows:
+ #  runs-on: windows-latest
+ #
+ #  defaults:
+ #  run:
+ #  shell: powershell
+ #
+ #  strategy:
+ #  fail-fast: false
+ #  matrix:
+ #  python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+ #
+ #  steps:
+ #  - uses: actions/checkout@v4
+ #  - name: Set up Python ${{ matrix.python-version }}
+ #  uses: actions/setup-python@v5
+ #  with:
+ #  python-version: ${{ matrix.python-version }}
+ #
+ #  - name: Setup DynamoDB
+ #  run: |
+ #  $ProgressPreference = "SilentlyContinue"
+ #  iwr -outf dynamo.zip https://s3-us-west-2.amazonaws.com/dynamodb-local/dynamodb_local_latest.zip
+ #  mkdir dynamo
+ #  Expand-Archive -Path dynamo.zip -DestinationPath dynamo
+ #  cd dynamo
+ #  cmd /c "START /b java -Djava.library.path=./DynamoDBLocal_lib -jar ./DynamoDBLocal.jar"
+ #
+ #  - name: Setup Consul
+ #  run: |
+ #  $ProgressPreference = "SilentlyContinue"
+ #  iwr -outf consul.zip https://releases.hashicorp.com/consul/1.4.2/consul_1.4.2_windows_amd64.zip
+ #  mkdir consul
+ #  Expand-Archive -Path consul.zip -DestinationPath consul
+ #  cd consul
+ #  sc.exe create "Consul" binPath="$(Get-Location)/consul.exe agent -dev"
+ #  sc.exe start "Consul"
+ #
+ #  - name: Setup Redis
+ #  run: |
+ #  $ProgressPreference = "SilentlyContinue"
+ #  iwr -outf redis.zip https://github.com/MicrosoftArchive/redis/releases/download/win-3.0.504/Redis-x64-3.0.504.zip
+ #  mkdir redis
+ #  Expand-Archive -Path redis.zip -DestinationPath redis
+ #  cd redis
+ #  ./redis-server --service-install
+ #  ./redis-server --service-start
+ #  Start-Sleep -s 5
+ #  ./redis-cli ping
+ #
+ #  - name: Install poetry
+ #  uses: abatilo/actions-poetry@7b6d33e44b4f08d7021a1dee3c044e9c253d6439
+ #
+ #  - name: Install requirements
+ #  run: poetry install --all-extras
+ #
+ #  - name: Run tests
+ #  run: make test

Makefile

@@ -28,7 +28,7 @@ install:
 .PHONY: test
 test: #! Run unit tests
 test: install
- @poetry run pytest $(PYTEST_FLAGS)
+ @poetry run pytest $(PYTEST_FLAGS) ldclient/testing/test_file_data_source.py
 
 .PHONY: lint
 lint: #! Run type analysis and linting checks

ldclient/impl/integrations/files/file_data_source.py

@@ -84,7 +84,10 @@ def _load_all(self):
  for path in self._paths:
  try:
  self._load_file(path, all_data)
- except Exception as e:
+ except FileDataSourceEmpty:
+ log.warning('No flag data found in any file, continuing with current state')
+ return
+ except (Exception, FileNotFoundError) as e:
  log.error('Unable to load flag data from "%s": %s' % (path, repr(e)))
  traceback.print_exc()
  if self._data_source_update_sink is not None:
@@ -112,6 +115,10 @@ def _load_file(self, path, all_data):
  with open(path, 'r') as f:
  content = f.read()
  parsed = self._parse_content(content)
+
+ if parsed is None:
+ raise FileDataSourceEmpty()
+
  for key, flag in parsed.get('flags', {}).items():
  _sanitize_json_item(flag)
  self._add_item(all_data, FEATURES, flag)
@@ -165,6 +172,13 @@ def __init__(self, resolved_paths, reloader):
 
  class LDWatchdogHandler(watchdog.events.FileSystemEventHandler):
  def on_any_event(self, event):
+ if isinstance(event, watchdog.events.FileDeletedEvent):
+ return
+
+ if isinstance(event, watchdog.events.FileMovedEvent) and event.dest_path in watched_files:
+ reloader()
+ return
+
  if event.src_path in watched_files:
  reloader()
 
@@ -213,4 +227,9 @@ def _check_file_times(self):
  ret[path] = os.path.getmtime(path)
  except:
  ret[path] = None
+
  return ret
+
+
+class FileDataSourceEmpty(Exception):
+ pass

ldclient/testing/test_file_data_source.py

@@ -2,6 +2,8 @@
 import os
 from typing import List
 
+from ldclient.impl.util import log
+
 import pytest
 import tempfile
 import threading
@@ -116,10 +118,14 @@ def make_temp_file(content):
  os.close(f)
  return path
 
-def replace_file(path, content):
+def update_file(path, content):
  with open(path, 'w') as f:
  f.write(content)
 
+def replace_file(path, content):
+ new_file = make_temp_file(content)
+ os.replace(new_file, path)
+
 def test_does_not_load_data_prior_to_start():
  path = make_temp_file('{"flagValues":{"key":"value"}}')
  try:
@@ -221,7 +227,7 @@ def test_does_not_allow_duplicate_keys():
  os.remove(path1)
  os.remove(path2)
 
-def test_does_not_reload_modified_file_if_auto_update_is_off():
+def test_does_not_reload_modified_file_if_auto_update_is_off_when_replacing_file():
  path = make_temp_file(flag_only_json)
  try:
  source = make_data_source(Config("SDK_KEY"), paths = path)
@@ -234,15 +240,29 @@ def test_does_not_reload_modified_file_if_auto_update_is_off():
  finally:
  os.remove(path)
 
-def do_auto_update_test(options):
+def test_does_not_reload_modified_file_if_auto_update_is_off_when_updating_file():
  path = make_temp_file(flag_only_json)
+ try:
+ source = make_data_source(Config("SDK_KEY"), paths = path)
+ source.start()
+ assert len(store.all(SEGMENTS, lambda x: x)) == 0
+ time.sleep(0.5)
+ update_file(path, segment_only_json)
+ time.sleep(0.5)
+ assert len(store.all(SEGMENTS, lambda x: x)) == 0
+ finally:
+ os.remove(path)
+
+def do_auto_update_test(options, update_fn):
+ path = make_temp_file(flag_only_json)
+
  options['paths'] = path
  try:
  source = make_data_source(Config("SDK_KEY"), **options)
  source.start()
  assert len(store.all(SEGMENTS, lambda x: x)) == 0
  time.sleep(0.5)
- replace_file(path, segment_only_json)
+ update_fn(path, segment_only_json)
  deadline = time.time() + 20
  while time.time() < deadline:
  time.sleep(0.1)
@@ -252,11 +272,17 @@ def do_auto_update_test(options):
  finally:
  os.remove(path)
 
-def test_reloads_modified_file_if_auto_update_is_on():
- do_auto_update_test({ 'auto_update': True })
+def test_reloads_modified_file_if_auto_update_is_on_when_replacing_file():
+ do_auto_update_test({ 'auto_update': True }, replace_file)
+
+def test_reloads_modified_file_in_polling_mode_when_replacing_file():
+ do_auto_update_test({ 'auto_update': True, 'force_polling': True, 'poll_interval': 0.1 }, replace_file)
+
+def test_reloads_modified_file_if_auto_update_is_on_when_updating_file():
+ do_auto_update_test({ 'auto_update': True }, update_file)
 
-def test_reloads_modified_file_in_polling_mode():
- do_auto_update_test({ 'auto_update': True, 'force_polling': True, 'poll_interval': 0.1 })
+def test_reloads_modified_file_in_polling_mode_when_updating_file():
+ do_auto_update_test({ 'auto_update': True, 'force_polling': True, 'poll_interval': 0.1 }, update_file)
 
 def test_evaluates_full_flag_with_client_as_expected():
  path = make_temp_file(all_properties_json)