Merge remote-tracking branch 'la-vache/main' into L2/23-219

unicode-org · Dec 29, 2023 · dac3cf6 · dac3cf6
2 parents cbdbdd8 + b753a41
commit dac3cf6
Show file tree

Hide file tree

Showing 103 changed files with 500,005 additions and 408,548 deletions.
diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
@@ -0,0 +1,87 @@
+name: Pipeline
+
+
+on:
+  pull_request:
+    branches: ['*']
+    types: [opened, synchronize, reopened, labeled, unlabeled, ready_for_review, converted_to_draft, edited]
+
+jobs:
+  labels-for-repertoire-changes:
+    name: Labels for repertoire changes
+    outputs:
+      repertoire-changed: ${{ steps.compare-repertoire.outputs.repertoire-changed }}
+      pipeline-label: ${{ steps.check-labels.outputs.pipeline-label }}
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout merged UnicodeData.txt
+      uses: actions/checkout@v3
+      with:
+        path: merged
+        sparse-checkout: unicodetools/data/ucd/dev/UnicodeData.txt
+    - name: Checkout base UnicodeData.txt
+      uses: actions/checkout@v3
+      with:
+        ref: ${{ github.event.pull_request.base.sha }}
+        path: base
+        sparse-checkout: unicodetools/data/ucd/dev/UnicodeData.txt
+    - name: Compare repertoire
+      id: compare-repertoire
+      run: |
+        # Look for changes affecting the first two fields of UnicodeData.txt (code point and name).
+        sed 's/^\([^;]*;[^;]*\);.*$/\1/' merged/unicodetools/data/ucd/dev/UnicodeData.txt > merged-repertoire.txt
+        sed 's/^\([^;]*;[^;]*\);.*$/\1/' base/unicodetools/data/ucd/dev/UnicodeData.txt > base-repertoire.txt
+        if diff base-repertoire.txt merged-repertoire.txt
+        then echo "repertoire-changed=false" >> "$GITHUB_OUTPUT"
+        else echo "repertoire-changed=true" >> "$GITHUB_OUTPUT"
+        fi
+    - name: Checkout Python scripts
+      uses: actions/checkout@v3
+      with:
+        sparse-checkout: py/pipeline-workflow
+    - name: Check pipeline labels
+      id: check-labels
+      if: steps.compare-repertoire.outputs.repertoire-changed == 'true'
+      run: python3 py/pipeline-workflow/compare-repertoire.py
+  l2-document:
+    needs: labels-for-repertoire-changes
+    if: ${{ always() && needs.labels-for-repertoire-changes.outputs.repertoire-changed == 'true' }}
+    name: Proposal document
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout Python scripts
+      uses: actions/checkout@v3
+      with:
+        sparse-checkout: py/pipeline-workflow
+    - name: Check L2 document
+      run: |
+        python3 py/pipeline-workflow/check-l2-document.py
+  utc-decision:
+    needs: labels-for-repertoire-changes
+    if: needs.labels-for-repertoire-changes.outputs.repertoire-changed == 'true' && needs.labels-for-repertoire-changes.outputs.pipeline-label != 'pipeline-recommended-to-UTC'
+    name: UTC decision
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout Python scripts
+      uses: actions/checkout@v3
+      with:
+        sparse-checkout: py/pipeline-workflow
+    - name: Check UTC decision
+      run: python3 py/pipeline-workflow/check-utc-decision.py
+  draft-unless-approved:
+    needs: labels-for-repertoire-changes
+    if: needs.labels-for-repertoire-changes.outputs.repertoire-changed == 'true'
+    name: Draft unless approved
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout Python scripts and DerivedAge.txt
+      uses: actions/checkout@v3
+      with:
+        sparse-checkout: |
+          py/pipeline-workflow
+          unicodetools/data/ucd/dev/DerivedAge.txt
+    - name: Check draft status
+      env:
+        PIPELINE_LABEL: ${{ needs.labels-for-repertoire-changes.outputs.pipeline-label }}
+      run: python3 py/pipeline-workflow/check-draft-status.py
+
diff --git a/UnicodeJsps/src/main/java/org/unicode/jsp/ScriptTester.java b/UnicodeJsps/src/main/java/org/unicode/jsp/ScriptTester.java
@@ -7,6 +7,7 @@
 import com.ibm.icu.lang.UScript;
 import com.ibm.icu.text.Normalizer;
 import com.ibm.icu.text.UnicodeSet;
+import java.util.ArrayList;
 import java.util.BitSet;
 import java.util.Collection;
 import java.util.Comparator;
@@ -20,6 +21,7 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.logging.Logger;
 import java.util.regex.Pattern;
+import org.unicode.props.UnicodeProperty;
 
 /**
  * Class for testing whether strings have allowed combinations of multiple scripts.
@@ -319,6 +321,7 @@ public static class ScriptExtensions {
         public static final Comparator<BitSet> COMPARATOR =
                 new Comparator<BitSet>() {
 
+                    @Override
                     public int compare(BitSet o1, BitSet o2) {
                         int diff = o1.cardinality() - o2.cardinality();
                         if (diff != 0) return diff;
@@ -344,6 +347,7 @@ private static class MyHandler extends FileUtilities.SemiFileReader {
 
             UnicodeMap<BitSet> map = new UnicodeMap<BitSet>();
 
+            @Override
             public boolean handleLine(int start, int end, String[] items) {
                 BitSet bitSet = new BitSet(LIMIT);
                 for (String script : SPACES.split(items[1])) {
@@ -429,21 +433,39 @@ public static UnicodeMap<String> getScriptSpecialsNames() {
         return result;
     }
 
-    public static String[][] getScriptSpecialsAlternates() {
+    public static String[][] getScriptSpecialsAlternates(UnicodeProperty scriptProp) {
         Collection<BitSet> availableValues = getScriptSpecials().getAvailableValues();
-        String[][] result = new String[availableValues.size()][];
+        List<String[]> result = new ArrayList<>();
         Set<String> names = new TreeSet<String>(); // to alphabetize
 
-        int i = 0;
         for (BitSet value : availableValues) {
             String baseName =
                     ScriptExtensions.getNames(value, UProperty.NameChoice.LONG, ",", names);
             String altName =
                     ScriptExtensions.getNames(value, UProperty.NameChoice.SHORT, ",", names);
             String[] row = {baseName, altName};
-            result[i++] = row;
+            result.add(row);
         }
-        return result;
+
+        // Get the single values, and build alternate values for the property, for isValidValue
+        // of a single script (eg Arab)
+        List<String> values = scriptProp.getAvailableValues();
+        for (String value : values) {
+            List<String> row = new ArrayList<>();
+            row.add(value);
+            for (String alias : scriptProp.getValueAliases(value)) {
+                if (!alias.equals(value)) {
+                    row.add(alias);
+                }
+            }
+            // duplicate it whenever singular, because the tooling expects at least 2 values (ugg)
+            if (row.size() == 1) {
+                row.add(value);
+            }
+            result.add(row.toArray(new String[row.size()]));
+        }
+
+        return result.toArray(new String[result.size()][]);
     }
 
     private ScriptTester(UnicodeMap<BitSet> character_scripts) {

diff --git a/UnicodeJsps/src/main/java/org/unicode/jsp/UnicodeSetUtilities.java b/UnicodeJsps/src/main/java/org/unicode/jsp/UnicodeSetUtilities.java
@@ -150,6 +150,7 @@ public MySymbolTable() {
         //      return null;
         //    }
 
+        @Override
         public boolean applyPropertyAlias(
                 String propertyName, String propertyValue, UnicodeSet result) {
             boolean status = false;
@@ -201,9 +202,11 @@ public boolean applyPropertyAlias(
                     }
                     ;
                     if (!status) {
-                        try {
-                            status = applyPropertyAlias0(prop, "No", result, !invert);
-                        } catch (Exception e) {
+                        if (prop.isType(UnicodeProperty.BINARY_OR_ENUMERATED_OR_CATALOG_MASK)) {
+                            try {
+                                status = applyPropertyAlias0(prop, "No", result, !invert);
+                            } catch (Exception e) {
+                            }
                         }
                         ;
                         if (!status) {
@@ -336,6 +339,7 @@ public ComparisonMatcher(String pattern, Relation comparator) {
             this.pattern = pattern;
         }
 
+        @Override
         public boolean test(String value) {
             int comp = comparator.compare(pattern, value.toString());
             switch (relation) {
@@ -352,6 +356,7 @@ public boolean test(String value) {
             }
         }
 
+        @Override
         public PatternMatcher set(String pattern) {
             this.pattern = pattern;
             return this;