ICU-23197 rework J DateFormatSymbols eraNames loading as in C (#3634)

- Also remove related logKnownIssue test skips, update tests for Meiji era start

Author: pedberg-icu

icu4j/main/core/src/main/java/com/ibm/icu/impl/EraRules.java

@@ -40,10 +40,14 @@ private EraRules(int[] startDates, int minEra, int numEras) {
     }
 
     public static EraRules getInstance(CalType calType, boolean includeTentativeEra) {
+        return getInstance(calType.getId(), includeTentativeEra);
+    }
+
+    public static EraRules getInstance(String calId, boolean includeTentativeEra) {
         UResourceBundle supplementalDataRes = UResourceBundle.getBundleInstance(ICUData.ICU_BASE_NAME,
                 "supplementalData", ICUResourceBundle.ICU_DATA_CLASS_LOADER);
         UResourceBundle calendarDataRes = supplementalDataRes.get("calendarData");
-        UResourceBundle calendarTypeRes = calendarDataRes.get(calType.getId());
+        UResourceBundle calendarTypeRes = calendarDataRes.get(calId);
         UResourceBundle erasRes = calendarTypeRes.get("eras");
 
         int numEras = erasRes.getSize();
@@ -58,10 +62,10 @@ public static EraRules getInstance(CalType calType, boolean includeTentativeEra)
             try {
                 eraIdx = Integer.parseInt(eraIdxStr);
             } catch (NumberFormatException e) {
-                throw new ICUException("Invalid era rule key:" + eraIdxStr + " in era rule data for " + calType.getId());
+                throw new ICUException("Invalid era rule key:" + eraIdxStr + " in era rule data for " + calId);
             }
             if (eraIdx < 0) {
-                throw new ICUException("Era rule key:" + eraIdxStr + " in era rule data for " + calType.getId()
+                throw new ICUException("Era rule key:" + eraIdxStr + " in era rule data for " + calId
                         + " must be >= 0");
             }
             if (eraIdx + 1 > eraStartDates.size()) {
@@ -74,7 +78,7 @@ public static EraRules getInstance(CalType calType, boolean includeTentativeEra)
             // Now set the startDate that we just read
             if (isSet(eraStartDates.get(eraIdx).intValue())) {
                 throw new ICUException(
-                        "Duplicated era rule for rule key:" + eraIdxStr + " in era rule data for " + calType.getId());
+                        "Duplicated era rule for rule key:" + eraIdxStr + " in era rule data for " + calId);
             }
 
             boolean hasName = true;
@@ -88,7 +92,7 @@ public static EraRules getInstance(CalType calType, boolean includeTentativeEra)
                     if (fields.length != 3 || !isValidRuleStartDate(fields[0], fields[1], fields[2])) {
                         throw new ICUException(
                                 "Invalid era rule date data:" + Arrays.toString(fields) + " in era rule data for "
-                                + calType.getId());
+                                + calId);
                     }
                     eraStartDates.set(eraIdx, encodeDate(fields[0], fields[1], fields[2]));
                 } else if (key.equals("named")) {
@@ -112,7 +116,7 @@ public static EraRules getInstance(CalType calType, boolean includeTentativeEra)
                     eraStartDates.set(eraIdx, MIN_ENCODED_START);
                 } else {
                     throw new ICUException("Missing era start/end rule date for key:" + eraIdxStr + " in era rule data for "
-                            + calType.getId());
+                            + calId);
                 }
             }
 

icu4j/main/core/src/main/java/com/ibm/icu/text/DateFormatSymbols.java

@@ -26,6 +26,7 @@
 
 import com.ibm.icu.impl.CacheBase;
 import com.ibm.icu.impl.CalendarUtil;
+import com.ibm.icu.impl.EraRules;
 import com.ibm.icu.impl.ICUData;
 import com.ibm.icu.impl.ICUResourceBundle;
 import com.ibm.icu.impl.SoftCache;
@@ -1806,37 +1807,8 @@ protected void processResource(String path, UResource.Key key, UResource.Value v
                     String[] dataArray = value.getStringArray();
                     arrays.put(currentPath, dataArray);
                 } else if (value.getType() == ICUResourceBundle.TABLE) {
-                    // We might have an eras table that is replacing an eras leaf array
-                    if (currentPath.startsWith("eras")) {
-                        // path is one of eras/wide, eras/abbreviated, eras/narrow
-                        UResource.Table rDataTable = value.getTable();
-                        int dataTableSize = rDataTable.getSize();
-                        ArrayList<String> dataList = new ArrayList<>(dataTableSize);
-                        // Expand the ArrayList as necessary to have index from 0 up to the max
-                        // eraCode, and fill in the slots for the eras defined in the resource data
-                        // (other slots get nulls).
-                        for (int dataTableIndex = 0; dataTableIndex < dataTableSize; dataTableIndex++) {
-                            rDataTable.getKeyAndValue(dataTableIndex, key, value);
-                            int listIndex = Integer.parseInt(key.toString());
-                            if (listIndex + 1 > dataList.size()) {
-                                dataList.ensureCapacity(listIndex + 1); // needed only to minimize expansions
-                                // Fill in empty strings for all added slots
-                                while (dataList.size() < listIndex + 1) {
-                                    dataList.add("");
-                                }
-                            }
-                            // Now set the eraName that we just read
-                            String eraName = (value.getType() == ICUResourceBundle.STRING) ? value.getString() : "";
-                            dataList.set(listIndex, eraName);
-                        }
-                        // Now convert to array
-                        String[] dataArray = dataList.toArray(new String[dataList.size()]);
-                        // Save the array
-                        arrays.put(currentPath, dataArray);
-                    } else {
-                        // We are not on a leaf, recursively process the subtable.
-                        processResource(currentPath, key, value);
-                    }
+                    // We are not on a leaf, recursively process the subtable.
+                    processResource(currentPath, key, value);
                 }
             }
         }
@@ -1898,6 +1870,46 @@ private DateFormatSymbols(ULocale desiredLocale, ICUResourceBundle b, String cal
         initializeData(desiredLocale, b, calendarType);
     }
 
+    /**
+     * Convert era names map from CalendarSink to array, filling in missing values from fallback.
+     * @internal
+     * @deprecated This API is ICU internal only.
+     */
+    protected String[] initEras(String erasKey, Map<String, Map<String, String>> maps,
+            ICUResourceBundle calBundle, int maxEra) {
+        Map<String, String> eraNamesTable = maps.get(erasKey);
+        if (eraNamesTable == null) {
+            return null;
+        }
+        ICUResourceBundle calErasWidthBundle = calBundle.findWithFallback(erasKey);
+        String[] eraArray = new String[maxEra + 1];
+        if (eraArray != null) {
+            for (int eraCode = 0; eraCode <= maxEra; eraCode++) {
+                String eraKey = Integer.toString(eraCode);
+                String eraName = eraNamesTable.get(eraKey);
+                if (eraName != null) {
+                    eraArray[eraCode] = eraName;
+                } else {
+                    // For a map, the sink does not seem to fill in parent entries for keys
+                    // that do not exist in the current bundle, that is why we need to explicitly
+                    // fill these in. Also true in ICU4C. Also pre-set to empty string in case
+                    // there is no parent entry.
+                    eraArray[eraCode] = "";
+                    if (calErasWidthBundle != null) {
+                        ICUResourceBundle calErasWidthKeyBundle = calErasWidthBundle.findWithFallback(eraKey);
+                        if (calErasWidthKeyBundle != null) {
+                            eraName = calErasWidthKeyBundle.getString();
+                            if (eraName != null) {
+                                eraArray[eraCode] = eraName;
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        return eraArray;
+    }
+
     /**
      * Initializes format symbols for the locale and calendar type
      * @param desiredLocale The locale whose symbols are desired.
@@ -1916,6 +1928,8 @@ protected void initializeData(ULocale desiredLocale, ICUResourceBundle b, String
             b = (ICUResourceBundle) UResourceBundle
                     .getBundleInstance(ICUData.ICU_BASE_NAME, desiredLocale);
         }
+        // Save the calendarType (with fallback) for later use with initEras:
+        String calTypeForEras = ((calendarType!=null)? calendarType : "gregorian");
 
         // Iterate over the resource bundle data following the fallbacks through different calendar types
         while (calendarType != null) {
@@ -1952,9 +1966,23 @@ protected void initializeData(ULocale desiredLocale, ICUResourceBundle b, String
         Map<String, String[]> arrays = calendarSink.arrays;
         Map<String, Map<String, String>> maps = calendarSink.maps;
 
-        eras = arrays.get("eras/abbreviated");
-        eraNames = arrays.get("eras/wide");
-        narrowEras = arrays.get("eras/narrow");
+        // Era setup: Get maxEra from EraRules, get the calendar's era bundle:
+        EraRules eraRules = null;
+        try {
+            eraRules = EraRules.getInstance(calTypeForEras, false);
+        } catch (MissingResourceException e) {
+            // call IDs unsupported in supplmental era rules such as
+            // "iso8601" or bogus "unknown"; fix for here and for
+            // calBundle:
+            calTypeForEras = "gregorian";
+            eraRules = EraRules.getInstance(calTypeForEras, false);
+        }
+        int maxEra = (eraRules != null)? eraRules.getMaxEraCode() : 0;
+        ICUResourceBundle calBundle = b.findWithFallback("calendar/" + calTypeForEras);
+
+        eras = initEras("eras/abbreviated", maps, calBundle, maxEra);
+        eraNames = initEras("eras/wide", maps, calBundle, maxEra);
+        narrowEras = initEras("eras/narrow", maps, calBundle, maxEra);
 
         months = arrays.get("monthNames/format/wide");
         shortMonths = arrays.get("monthNames/format/abbreviated");

icu4j/main/core/src/test/java/com/ibm/icu/dev/test/calendar/JapaneseTest.java

@@ -253,17 +253,13 @@ public void TestForceGannenNumbering() {
         }
         String testString2 = testFmt2.format(refDate);
         if (testString2.length() < 2 || testString2.charAt(1) != '1') {
-            if (!logKnownIssue("ICU-23182", "Japanese calendar formatting")) {
-                errln("Formatting year 1 in created numeric style, got " + testString2 + " but expected 2nd char to be 1");
-            }
+            errln("Formatting year 1 in created numeric style, got " + testString2 + " but expected 2nd char to be 1");
         }
         // Now switch the patterns and verify that Gannen use follows the pattern
         testFmt1.applyPattern(patNumr);
         testString1 = testFmt1.format(refDate);
         if (testString1.length() < 2 || testString1.charAt(1) != '1') { //
-            if (!logKnownIssue("ICU-23182", "Japanese calendar formatting")) {
-                errln("Formatting year 1 in applied numeric style, got " + testString1 + " but expected 2nd char to be 1");
-            }
+            errln("Formatting year 1 in applied numeric style, got " + testString1 + " but expected 2nd char to be 1");
         }
         testFmt2.applyPattern(patText);
         testString2 = testFmt2.format(refDate);
@@ -290,10 +286,10 @@ public void Test5345parse() {
         fmt.applyPattern("G y");
         logln("fmt's locale = " + fmt.getLocale(ULocale.ACTUAL_LOCALE));
         //SimpleDateFormat fmt = new SimpleDateFormat("G y", new ULocale("en_US@calendar=japanese"));
-        long aDateLong = -3197117222000L; // 1868-09-08 00:00 Pacific Time (GMT-07:52:58)
+        long aDateLong = -3193229222000L; // 1868-10-23 00:00 Pacific Time (GMT-07:52:58)
         if (TimeZone.getDefaultTimeZoneType() == TimeZone.TIMEZONE_JDK) {
             // Java time zone implementation does not support LMTs
-            aDateLong = -3197116800000L; // 1868-09-08 00:00 Pacific Time (GMT-08:00)
+            aDateLong = -3193228800000L; // 1868-10-23 00:00 Pacific Time (GMT-08:00)
         }
         Date aDate = new Date(aDateLong);
         logln("aDate: " + aDate.toString() +", from " + aDateLong);
@@ -304,9 +300,7 @@ public void Test5345parse() {
         logln("as Japanese Calendar: " + str);
         String expected = "Meiji 1";
         if(!str.equals(expected)) {
-            if (!logKnownIssue("ICU-23186", "Japanese calendar round trip fails")) {
-                errln("FAIL: Expected " + expected + " but got " + str);
-            }
+            errln("FAIL: Expected " + expected + " but got " + str);
         }
         Date otherDate;
         try {
@@ -319,10 +313,8 @@ public void Test5345parse() {
                 long oLong = otherDate.getTime();
                 long aLong = otherDate.getTime();
 
-                if (!logKnownIssue("ICU-23186", "Japanese calendar round trip fails")) {
-                    errln("FAIL: Parse incorrect of " + expected + ":  wanted " + aDate + " ("+aLong+"), but got " +  " " +
-                        otherDate + " ("+oLong+") = " + str3 + " not " + dd.toString() );
-                }
+                errln("FAIL: Parse incorrect of " + expected + ":  wanted " + aDate + " ("+aLong+"), but got " +  " " +
+                    otherDate + " ("+oLong+") = " + str3 + " not " + dd.toString() );
             } else {
                 logln("Parsed OK: " + expected);
             }

icu4j/main/core/src/test/java/com/ibm/icu/dev/test/format/DateIntervalFormatTest.java

@@ -884,10 +884,6 @@ private void expect(String[] data, int data_length) {
             String expected = data[i++];
             String formatted = dtitvfmt.format(dtitv);
             if ( !formatted.equals(Utility.unescape(expected)) )  {
-                if (locName.equals("ja-u-ca-japanese") &&
-                    logKnownIssue("ICU-23182", "Japanese calendar formatting")) {
-                    continue;
-                }
                 if (locName.equals("de") && 
                     ( oneSkeleton.equals("hv") || oneSkeleton.equals("hz") ) &&
                     logKnownIssue("ICU-23185", "Date time formatting with hz and hv needs revisiting")) {
@@ -2457,10 +2453,6 @@ public void testTicket21222ROCEraDiff() {
 
     @Test
     public void testTicket21222JapaneseEraDiff() {
-        if (logKnownIssue("ICU-23182", "Japanese calendar formatting")) {
-            return;
-        }
-    
         Calendar cal = Calendar.getInstance(TimeZone.GMT_ZONE);
         DateIntervalFormat japanese = DateIntervalFormat.getInstance(
             "h", new ULocale("ja@calendar=japanese"));

icu4j/main/core/src/test/java/com/ibm/icu/dev/test/format/DateTimeGeneratorTest.java

@@ -343,11 +343,6 @@ public void TestBasic() {
                 if (GENERATE_TEST_DATA) {
                     logln("new String[] {\"" + testSkeleton + "\", \"" + Utility.escape(formatted) + "\"},");
                 } else if (!formatted.equals(testFormatted)) {
-                    if (uLocale.getName().equals("ja@calendar=japanese") && 
-                        (testSkeleton.equals("yM") || testSkeleton.equals("yMd")) &&
-                        logKnownIssue("ICU-23182", "Japanese calendar formatting") ) {
-                        continue;
-                    }
                     errln(uLocale + "\tformatted string doesn't match test case: " + testSkeleton + "\t generated: " +  pattern + "\t expected: " + testFormatted + "\t got: " + formatted);
                     if (true) { // debug
                         pattern = dtfg.getBestPattern(testSkeleton);