Tests inversion salaires

.github/has-functional-changes.sh

@@ -3,7 +3,7 @@
 IGNORE_DIFF_ON="README.md CONTRIBUTING.md Makefile .gitignore .github/*"
 
 last_tagged_commit=`git describe --tags --abbrev=0 --first-parent`  # --first-parent ensures we don't follow tags not published in master through an unlikely intermediary merge commit
-
+echo ".github/has-functional-changes.sh : last_tagged_commit=$last_tagged_commit"
 if git diff-index --name-only --exit-code $last_tagged_commit -- . `echo " $IGNORE_DIFF_ON" | sed 's/ / :(exclude)/g'`  # Check if any file that has not be listed in IGNORE_DIFF_ON has changed since the last tag was published.
 then
   echo "No functional changes detected."

.gitlab-ci.yml

@@ -1,4 +1,4 @@
-include: 'gitlab_ci/all_years_build_and_aggregates.yml'
+include: '.gitlab-ci/all_years_build_and_aggregates.yml'
 
 variables:
   PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
@@ -7,6 +7,7 @@ variables:
   # OUT_FOLDER: "$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA" # For branch-commit_id
   OUT_FOLDER: "$CI_COMMIT_REF_NAME" # For just branch
   ROOT_FOLDER: "/mnt/data-out/openfisca-france-data"
+#  GIT_DEPTH: 1000 # To be able to get last tag (default 50)
 
 cache:
   paths:
@@ -25,14 +26,12 @@ stages:
   - aggregates_all
   - anaconda
 
-
 before_script:
   # To be sure we are up to date even if we do not rebuild docker image
   - make install
-  - cp ./ci-runner/openfisca_survey_manager_raw_data.ini ~/.config/openfisca-survey-manager/raw_data.ini
+  - cp ./.gitlab-ci/openfisca_survey_manager_raw_data.ini ~/.config/openfisca-survey-manager/raw_data.ini
   - echo "End of before_script"
 
-
 build docker image:
   stage: docker
   tags:
@@ -51,7 +50,6 @@ build docker image:
   # Build Docker is needed only if code as changed.
   when: manual
 
-
 test:
   image: $CI_REGISTRY_IMAGE:latest
   script:
@@ -60,7 +58,6 @@ test:
   tags:
   - openfisca
 
-
 clean_folder:
   before_script:
     - ''
@@ -70,15 +67,14 @@ clean_folder:
   - openfisca
   when: manual
 
-
 build_collection:
   image: $CI_REGISTRY_IMAGE:latest
   script:
   - echo "Begin with fresh config"
   - rm -rf $ROOT_FOLDER/$OUT_FOLDER || true
   - mkdir -p $ROOT_FOLDER/$OUT_FOLDER/data_collections/
   - mkdir -p $ROOT_FOLDER/$OUT_FOLDER/data_output/
-  - cp ./ci-runner/openfisca_survey_manager_config.ini ~/.config/openfisca-survey-manager/config.ini
+  - cp ./.gitlab-ci/openfisca_survey_manager_config.ini ~/.config/openfisca-survey-manager/config.ini
   - echo "Custom output folder"
   - sed -i "s/BRANCH_NAME/$OUT_FOLDER/" ~/.config/openfisca-survey-manager/config.ini
   - 'echo "{\"name\": \"erfs_fpr\", \"surveys\": {}}" > $ROOT_FOLDER/$OUT_FOLDER/data_collections/erfs_fpr.json'
@@ -94,7 +90,6 @@ build_collection:
   - openfisca
   when: manual
 
-
 copy_previous_build_collections:
   before_script:
     - ''
@@ -108,7 +103,7 @@ copy_previous_build_collections:
         cp $ROOT_FOLDER/master/openfisca_survey_manager_config-after-build-collection.ini $ROOT_FOLDER/$OUT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini
         sed -i "s/master/$OUT_FOLDER/" $ROOT_FOLDER/$OUT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini
         cp $ROOT_FOLDER/master/data_collections/erfs_fpr.json $ROOT_FOLDER/$OUT_FOLDER/data_collections/erfs_fpr.json
-        cp ./ci-runner/empty_openfisca_erfs_fpr.json $ROOT_FOLDER/$OUT_FOLDER/data_collections/openfisca_erfs_fpr.json
+        cp ./.gitlab-ci/empty_openfisca_erfs_fpr.json $ROOT_FOLDER/$OUT_FOLDER/data_collections/openfisca_erfs_fpr.json
     fi
   stage: build_collection
   tags:
@@ -128,19 +123,25 @@ diagnostics:
   - cp $ROOT_FOLDER/$OUT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2019.ini
     ~/.config/openfisca-survey-manager/config.ini
   - mkdir -p ~/.config/openfisca-france-data
-  - cp ./ci-runner/openfisca_france_data_config.ini
+  - cp ./.gitlab-ci/openfisca_france_data_config.ini
     ~/.config/openfisca-france-data/config.ini
   - sed -i "s/BRANCH_NAME/$OUT_FOLDER/" ~/.config/openfisca-france-data/config.ini
   - cat ~/.config/openfisca-france-data/config.ini
   - compare-erfs-fpr-input -u -s -v
   - cp -r /mnt/data-out/openfisca-france-data/$OUT_FOLDER/figures_directory .
-  - ls -alrth
-  - ls -alrth figures_directory
   - cp -r ./figures_directory $ROOT_FOLDER/$OUT_FOLDER/data_output
   stage: diagnostics
   tags:
   - openfisca
 
+check-version-and-changelog:
+  stage: diagnostics
+  before_script:
+  - ''
+  needs:
+  - input_data-2019
+  script:
+  - .gitlab-ci/is-version-number-acceptable.sh
 
 run_on_all_years:
   stage: run_on_all_years
@@ -151,15 +152,26 @@ run_on_all_years:
     - echo "On ne fait rien"
   when: manual
 
+check-for-functional-changes:
+  stage: anaconda
+  needs:
+  - check-version-and-changelog
+  before_script:
+  - ''
+  script:
+  - if `.github/has-functional-changes.sh` ; then echo "OK to build package" ; fi
+  only: 
+  - master
 
 build_conda_package:
+  stage: anaconda
+  needs:
+  - check-for-functional-changes
   before_script:
   - ''
-  except:
+  only:
   - master
   image: continuumio/miniconda3
   script:
   - conda install -y conda-build anaconda-client
-  - conda build -c conda-forge -c openfisca --token $ANACONDA_TOKEN --user OpenFisca
-    .conda
-  stage: anaconda
+  - conda build -c conda-forge -c openfisca --token $ANACONDA_TOKEN --user OpenFisca .conda

ci-runner/README.md → .gitlab-ci/README.md

@@ -7,7 +7,7 @@ This folder contains files needed for the CI.
 To separate the different years and survey files we have a script that build the CI script.
 
 ```
-python ci-runner/build_ci.py
+python .gitlab-ci/build_ci.py
 ```
 
 It will create the file `.gitlab-ci.yml` that is read by Gitlab Runner to execute the CI.
@@ -31,8 +31,8 @@ All following steps is run with this docker image.
 It is a manual step because it does not to be build each time and took a very long time : between 2 and 4 hours. It use the `build-collection` command from [OpenFisca-Survey-Manager](https://github.com/openfisca/openfisca-survey-manager).
 
 Input :
-- [../ci-runner/openfisca_survey_manager_config.ini](ci-runner/openfisca_survey_manager_config.ini)
-- [../ci-runner/openfisca_survey_manager_raw_data.ini](ci-runner/openfisca_survey_manager_raw_data.ini)
+- [../.gitlab-ci/openfisca_survey_manager_config.ini](.gitlab-ci/openfisca_survey_manager_config.ini)
+- [../.gitlab-ci/openfisca_survey_manager_raw_data.ini](.gitlab-ci/openfisca_survey_manager_raw_data.ini)
 - All survey's files located in `/mnt/data-in/erfs-fpr/` folder that is accessible to the CI Runner.
 
 Output :

gitlab_ci/all_years_build_and_aggregates.yml → .gitlab-ci/all_years_build_and_aggregates.yml

@@ -1,17 +1,16 @@
 
 ################################################
 # GENERATED FILE, DO NOT EDIT
-# Please visit ci-runner/README.md
+# Please visit .gitlab-ci/README.md
 ################################################
 
 input_data-2019:
   image: $CI_REGISTRY_IMAGE:latest
   script:
   - echo "build_input_data-2019"
   - mkdir -p $ROOT_FOLDER/$OUT_FOLDER
-  - cp $ROOT_FOLDER/$OUT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini ~/.config/openfisca-survey-manager/config.ini
-  - cat ~/.config/openfisca-survey-manager/config.ini
-  - ls $ROOT_FOLDER/$OUT_FOLDER/data_collections
+  - cp $ROOT_FOLDER/$OUT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini
+    ~/.config/openfisca-survey-manager/config.ini
   - build-erfs-fpr -y 2019 -f $ROOT_FOLDER/$OUT_FOLDER/data_output/erfs_flat_2019.h5
   - cp ~/.config/openfisca-survey-manager/config.ini $ROOT_FOLDER/$OUT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2019.ini
   stage: build_input_data

ci-runner/build_ci.py → .gitlab-ci/build_ci.py

@@ -3,20 +3,20 @@
 
 Run in project root folder:
 
-python ci-runner/build_ci.py
+python .gitlab-ci/build_ci.py
 """
 import configparser
 import yaml
 
 # Config file use to get the available years
-CONFIG = "./ci-runner/openfisca_survey_manager_raw_data.ini"
+CONFIG = "./.gitlab-ci/openfisca_survey_manager_raw_data.ini"
 
 
 def header():
     return """
 ################################################
 # GENERATED FILE, DO NOT EDIT
-# Please visit ci-runner/README.md
+# Please visit .gitlab-ci/README.md
 ################################################
 
 """
@@ -111,7 +111,6 @@ def get_erfs_years():
         raise KeyError
 
 
-
 def build_gitlab_ci(erfs_years):
     gitlab_ci = header()
     # gitlab_ci += yaml.dump(make_test())
@@ -134,7 +133,7 @@ def main():
     # For testing only some years
     # erfs_years = ["2016", "2017", "2018"]
     gitlab_ci = build_gitlab_ci(erfs_years)
-    with open(r"./gitlab_ci/all_years_build_and_aggregates.yml", mode="w") as file:
+    with open(r"./.gitlab-ci/all_years_build_and_aggregates.yml", mode="w") as file:
         file.write(gitlab_ci)
     print("Done with success!")
 

.gitlab-ci/has-functional-changes.sh

@@ -0,0 +1,18 @@
+#! /usr/bin/env bash
+
+IGNORE_DIFF_ON="README.md CONTRIBUTING.md Makefile .gitignore .github/*"
+
+# Fetch all tags
+git fetch --tags
+
+last_tagged_commit=`git tag --list | sort -V | grep -v ipp | tail -1`
+echo ".gitlab-ci/has-functional-changes.sh : avec git tag --list last_tagged_commit=$last_tagged_commit"
+
+# Check if any file that has not be listed in IGNORE_DIFF_ON has changed since the last tag was published.
+if git diff-index --name-only --exit-code $last_tagged_commit -- . `echo " $IGNORE_DIFF_ON" | sed 's/ / :(exclude)/g'`
+then
+  echo "No functional changes detected."
+  exit 1
+else
+  echo "The functional files above were changed."
+fi

.gitlab-ci/is-version-number-acceptable.sh

@@ -0,0 +1,47 @@
+#! /usr/bin/env bash
+
+if [[ ${CI_COMMIT_REF_NAME} == master ]]
+then
+    echo "No need for a version check on master.👍"
+    exit 0
+else
+    echo "Not on master."
+fi
+
+if ! $(dirname "$BASH_SOURCE")/has-functional-changes.sh
+then
+    echo "No need for a version update.👍"
+    exit 0
+else
+    echo "Need for a version update."
+fi
+
+current_version=`python setup.py --version`
+
+if git rev-parse --verify --quiet $current_version
+then
+    echo "Version $current_version already exists in commit:"
+    git --no-pager log -1 $current_version
+    echo
+    echo "Update the version number in setup.py before merging this branch into master.😒"
+    echo "Look at the CONTRIBUTING.md file to learn how the version number should be updated."
+    exit 1
+else
+    echo "Version $current_version don't exists in commit history 👍"
+fi
+
+if ! $(dirname "$BASH_SOURCE")/has-functional-changes.sh | grep --quiet CHANGELOG.md
+then
+    echo "CHANGELOG.md has not been modified, while functional changes were made. 😒"
+    echo "Explain what you changed before merging this branch into master."
+    echo "Look at the CONTRIBUTING.md file to learn how to write the changelog."
+    exit 2
+else
+    if ! grep --quiet $current_version  CHANGELOG.md
+    then
+        echo "CHANGELOG.md has been modified. BUT $current_version don't exists in it. 😒"
+        exit 3
+    else
+        echo "CHANGELOG.md has been modified and $current_version exists in it. 👍"
+    fi
+fi

ci-runner/openfisca_survey_manager_config.ini → .gitlab-ci/openfisca_survey_manager_config.ini

@@ -1,6 +1,6 @@
 # Template du fichier config.ini de openfisca-survey-manager
 # pour qu'il fonctionne avec oepnfisca-france-data
-# sur le ci-runner gitlab piloté par ipp/openfisca-france/data
+# sur le runner gitlab CI piloté par ipp/openfisca-france/data
 
 [collections]
 collections_directory = /mnt/data-out/openfisca-france-data/BRANCH_NAME/data_collections

ci-runner/openfisca_survey_manager_raw_data.ini → .gitlab-ci/openfisca_survey_manager_raw_data.ini

@@ -1,6 +1,6 @@
 # Template du fichier raw_data.ini de openfisca-survey-manager
 # pour qu'il fonctionne avec oepnfisca-france-data
-# sur le ci-runner gitlab piloté par ipp/openfisca-france/data
+# sur le runner GitLab CI piloté par ipp/openfisca-france/data
 
 ; [enquete_logement]
 ; 2006 = /home/ipp/data/enquete_logement/2006/stata

README.md

@@ -87,6 +87,12 @@ This should not display any error and end with:
 
 `Successfully installed OpenFisca-France-Data...`
 
+### Specifics due to Windows' handling of long paths
+
+On a Windows machine, the installation of Openfisca-France-Data may run into problems due to long path names, which Windows, by default, does not handle. These long paths are mostly inherited from the OpenFisca-France parameters, which are stored in a sometimes deeply nested folder.
+
+A possible workaround on Windows >= 10 is to lift the maximum path length limitation (as [indicated here](https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation?tabs=registry#enable-long-paths-in-windows-10-version-1607-and-later)).
+
 ## Execution
 
 Let's say that you would like to format `ERFS-FPR` survey data into OpenFisca formatted data.

openfisca_france_data/__init__.py

@@ -9,7 +9,7 @@
 import openfisca_france  # type: ignore
 
 # Load input variables and output variables into entities
-from openfisca_france_data.model import common, survey_variables, id_variables  # noqa analysis:ignore
+from openfisca_france_data.model import common, survey_variables  # noqa analysis:ignore
 from openfisca_france_data.model.base import * # noqa  analysis:ignore