From 5fdf098d07aa2b8ca9d1554cf569e4aa2f1de1c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 25 Nov 2024 21:34:57 +0100 Subject: [PATCH 001/170] fix: Update code --- .../table/AbstractEventJdbcTableManager.java | 13 ++++++------- .../JdbcValidationResultTableManager.java | 10 ++++------ .../DimensionIdentifierHelperTest.java | 19 ------------------- .../CommonRequestParamsMapperTest.java | 3 +-- .../query/NotConditionRendererTest.java | 1 - .../AnalyticsServiceReportingRateTest.java | 2 +- 6 files changed, 12 insertions(+), 36 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index be74348ffd8f..0b07dced4214 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -286,24 +286,23 @@ protected List getTrackedEntityAttributeColumns(Program pr * The select statement used by the table population. * * @param attribute the {@link TrackedEntityAttribute}. - * @param fromType the sql snippet related to "from" part + * @param columnExpression the column expression. * @param dataClause the data type related clause like "NUMERIC". * @return a select statement. */ protected String selectForInsert( - TrackedEntityAttribute attribute, String fromType, String dataClause) { + TrackedEntityAttribute attribute, String columnExpression, String dataClause) { return replaceQualify( """ - (select ${fromType} from ${trackedentityattributevalue} \ + (select ${columnExpression} from ${trackedentityattributevalue} \ where trackedentityid=en.trackedentityid \ - and trackedentityattributeid=${attributeId}\ - ${dataClause})\ + and trackedentityattributeid=${attributeId}${dataClause})\ ${closingParentheses} as ${attributeUid}""", Map.of( - "fromType", fromType, + "columnExpression", columnExpression, "dataClause", dataClause, "attributeId", String.valueOf(attribute.getId()), - "closingParentheses", getClosingParentheses(fromType), + "closingParentheses", getClosingParentheses(columnExpression), "attributeUid", quote(attribute.getUid()))); } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcValidationResultTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcValidationResultTableManager.java index 8d7ab2d85e0f..e25bd5756486 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcValidationResultTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcValidationResultTableManager.java @@ -31,7 +31,6 @@ import static org.hisp.dhis.commons.util.TextUtils.emptyIfTrue; import static org.hisp.dhis.commons.util.TextUtils.format; import static org.hisp.dhis.commons.util.TextUtils.removeLastComma; -import static org.hisp.dhis.commons.util.TextUtils.replace; import static org.hisp.dhis.db.model.DataType.CHARACTER_11; import static org.hisp.dhis.db.model.DataType.DATE; import static org.hisp.dhis.db.model.DataType.INTEGER; @@ -213,9 +212,9 @@ private List getDataYears(AnalyticsTableUpdateParams params) { String fromDateClause = params.getFromDate() == null ? "" - : replace( - "and ps.startdate >= '${fromDate}'", - Map.of("fromDate", DateUtils.toMediumDate(params.getFromDate()))); + : String.format( + " and ps.startdate >= '%s'", DateUtils.toMediumDate(params.getFromDate())); + String sql = replaceQualify( """ @@ -223,8 +222,7 @@ select distinct(extract(year from ps.startdate)) \ from ${validationresult} vrs \ inner join analytics_rs_periodstructure ps on vrs.periodid=ps.periodid \ where ps.startdate is not null \ - and vrs.created < '${startTime}' - ${fromDateClause}""", + and vrs.created < '${startTime}'${fromDateClause}""", Map.of( "startTime", toLongDate(params.getStartTime()), "fromDateClause", fromDateClause)); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/common/params/dimension/DimensionIdentifierHelperTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/common/params/dimension/DimensionIdentifierHelperTest.java index f4c5f57ce8b1..e45a6728764d 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/common/params/dimension/DimensionIdentifierHelperTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/common/params/dimension/DimensionIdentifierHelperTest.java @@ -35,17 +35,13 @@ import org.hisp.dhis.common.IllegalQueryException; import org.junit.jupiter.api.Test; -/** Unit tests for {@link DimensionIdentifierHelper} */ class DimensionIdentifierHelperTest { @Test void testFromFullDimensionId() { - // Given String fullDimensionId = "lxAQ7Zs9VYR[0].RaMbOrTys0n[4].bh1Edk21e2n"; - // When StringDimensionIdentifier dimensionIdentifier = fromFullDimensionId(fullDimensionId); - // Then assertEquals( "bh1Edk21e2n", dimensionIdentifier.getDimension().getUid(), @@ -64,12 +60,9 @@ void testFromFullDimensionId() { @Test void testFromDimensionIdWithUnsupportedOffset() { - // Given String dimensionIdWithDashOffset = "lxAQ7Zs9VYR[-].bh1Edk21e2n"; String dimensionIdWithStringOffset = "lxAQ7Zs9VYR[X].bh1Edk21e2n"; - // When - // Then assertThrows(IllegalQueryException.class, () -> fromFullDimensionId(dimensionIdWithDashOffset)); assertThrows( IllegalQueryException.class, () -> fromFullDimensionId(dimensionIdWithStringOffset)); @@ -77,13 +70,10 @@ void testFromDimensionIdWithUnsupportedOffset() { @Test void testFromFullDimensionIdWithSingleDimension() { - // Given String singleDimensionId = "bh1Edk21e2n"; - // When StringDimensionIdentifier dimensionIdentifier = fromFullDimensionId(singleDimensionId); - // Then assertEquals( "bh1Edk21e2n", dimensionIdentifier.getDimension().getUid(), @@ -96,13 +86,10 @@ void testFromFullDimensionIdWithSingleDimension() { @Test void testFromFullDimensionIdWithProgramAndDimension() { - // Given String singleDimensionId = "lxAQ7Zs9VYR.bh1Edk21e2n"; - // When StringDimensionIdentifier dimensionIdentifier = fromFullDimensionId(singleDimensionId); - // Then assertEquals( "bh1Edk21e2n", dimensionIdentifier.getDimension().getUid(), @@ -117,15 +104,12 @@ void testFromFullDimensionIdWithProgramAndDimension() { @Test void testFromFullDimensionIdWhenSingleDimensionHasOffset() { - // Given String singleDimensionWithOffset = "bh1Edk21e2n[2]"; - // When IllegalArgumentException thrown = assertThrows( IllegalArgumentException.class, () -> fromFullDimensionId(singleDimensionWithOffset)); - // Then assertEquals( "Only program and program stage can have offset", thrown.getMessage(), @@ -134,15 +118,12 @@ void testFromFullDimensionIdWhenSingleDimensionHasOffset() { @Test void testFromFullDimensionIdWhenDimensionHasInvalidFormat() { - // Given String invalidFullDimensionId = "lxAQ7Zs9VYR[1].RaMbOrTys0n[4].bh1Edk21e2n.invalid-id"; - // When IllegalArgumentException thrown = assertThrows( IllegalArgumentException.class, () -> fromFullDimensionId(invalidFullDimensionId)); - // Then assertEquals( "Invalid dimension identifier: " + invalidFullDimensionId, thrown.getMessage(), diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/common/processing/CommonRequestParamsMapperTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/common/processing/CommonRequestParamsMapperTest.java index a2657a931d0e..4b1e883c9dae 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/common/processing/CommonRequestParamsMapperTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/common/processing/CommonRequestParamsMapperTest.java @@ -435,8 +435,7 @@ void mapWhenProgramsCannotBeFound() { programService, dimensionIdentifierConverter); - // List has only one Program, but the CommonQueryRequest, below, has - // two. + // List has only one Program, but the CommonQueryRequest, has two List programs = List.of(program1); CommonRequestParams commonRequestParams = diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/common/query/NotConditionRendererTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/common/query/NotConditionRendererTest.java index 20de5f79ff9c..b3ab91830786 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/common/query/NotConditionRendererTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/common/query/NotConditionRendererTest.java @@ -32,7 +32,6 @@ import org.junit.jupiter.api.Test; class NotConditionRendererTest { - @Test void testInWithSingleValueProduceCorrectSql() { Renderable renderable = () -> "test"; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/AnalyticsServiceReportingRateTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/AnalyticsServiceReportingRateTest.java index 12f852c8e464..975fd51f7844 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/AnalyticsServiceReportingRateTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/AnalyticsServiceReportingRateTest.java @@ -102,7 +102,7 @@ void verifyReportingRatesValueWhenPeriodIsFilter() { DataQueryParams.newBuilder() .withOrganisationUnit(ou) // DATA ELEMENTS - .withDataElements(newArrayList(reportingRateA, reportingRateB, reportingRateC)) + .withDataElements(List.of(reportingRateA, reportingRateB, reportingRateC)) .withIgnoreLimit(true) // FILTERS (OU) .withFilters(List.of(new BaseDimensionalObject("pe", DimensionType.PERIOD, periods))) From 12f1f27f5236331791cf41086173ddb726ba5c28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 25 Nov 2024 21:41:04 +0100 Subject: [PATCH 002/170] fix: Update code --- .../data/DefaultDataQueryService.java | 2 +- ...er.java => DimensionalObjectProvider.java} | 2 +- .../outlier/data/OutlierQueryParser.java | 4 +-- .../service/AnalyticsOutlierService.java | 4 +-- ...aQueryServiceDimensionItemKeywordTest.java | 2 +- .../analytics/data/DataQueryServiceTest.java | 2 +- ...ava => DimensionalObjectProviderTest.java} | 6 ++--- .../data/OrgUnitTableJoinerTest.java | 24 ++++++++--------- .../analytics/data/QueryItemHelperTest.java | 16 ++++++------ .../SubexpressionPeriodOffsetUtilsTest.java | 26 +++++++++---------- .../data/handler/DataHandlerTest.java | 16 ------------ .../handler/SchemeIdResponseMapperTest.java | 8 +----- ...AbstractJdbcEventAnalyticsManagerTest.java | 1 + .../data/ColumnWithNullIfAndAliasTest.java | 6 ----- .../outlier/data/OutlierQueryParserTest.java | 4 +-- .../controller/AnalyticsControllerTest.java | 4 +-- 16 files changed, 50 insertions(+), 77 deletions(-) rename dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/{DimensionalObjectProducer.java => DimensionalObjectProvider.java} (99%) rename dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/{DimensionalObjectProducerTest.java => DimensionalObjectProviderTest.java} (99%) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultDataQueryService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultDataQueryService.java index 9d821881af0d..6ed9ff5c5838 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultDataQueryService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultDataQueryService.java @@ -93,7 +93,7 @@ @Service("org.hisp.dhis.analytics.DataQueryService") @RequiredArgsConstructor public class DefaultDataQueryService implements DataQueryService { - private final DimensionalObjectProducer dimensionalObjectProducer; + private final DimensionalObjectProvider dimensionalObjectProducer; private final IdentifiableObjectManager idObjectManager; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DimensionalObjectProducer.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DimensionalObjectProvider.java similarity index 99% rename from dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DimensionalObjectProducer.java rename to dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DimensionalObjectProvider.java index c776b35d49f0..c247113b657f 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DimensionalObjectProducer.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DimensionalObjectProvider.java @@ -116,7 +116,7 @@ */ @Component @RequiredArgsConstructor -public class DimensionalObjectProducer { +public class DimensionalObjectProvider { private final IdentifiableObjectManager idObjectManager; private final OrganisationUnitService organisationUnitService; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/outlier/data/OutlierQueryParser.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/outlier/data/OutlierQueryParser.java index a64dec95202b..65b350d257c9 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/outlier/data/OutlierQueryParser.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/outlier/data/OutlierQueryParser.java @@ -39,7 +39,7 @@ import java.util.UUID; import lombok.AllArgsConstructor; import org.apache.commons.lang3.StringUtils; -import org.hisp.dhis.analytics.data.DimensionalObjectProducer; +import org.hisp.dhis.analytics.data.DimensionalObjectProvider; import org.hisp.dhis.category.CategoryOptionCombo; import org.hisp.dhis.common.BaseDimensionalObject; import org.hisp.dhis.common.DisplayProperty; @@ -60,7 +60,7 @@ @AllArgsConstructor public class OutlierQueryParser { private final IdentifiableObjectManager idObjectManager; - private final DimensionalObjectProducer dimensionalObjectProducer; + private final DimensionalObjectProvider dimensionalObjectProducer; private final UserService userService; /** diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/outlier/service/AnalyticsOutlierService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/outlier/service/AnalyticsOutlierService.java index 9b7ed5e7a7d1..8dad2acf3de9 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/outlier/service/AnalyticsOutlierService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/outlier/service/AnalyticsOutlierService.java @@ -63,7 +63,7 @@ import org.hisp.dhis.analytics.cache.OutliersCache; import org.hisp.dhis.analytics.common.ColumnHeader; import org.hisp.dhis.analytics.common.TableInfoReader; -import org.hisp.dhis.analytics.data.DimensionalObjectProducer; +import org.hisp.dhis.analytics.data.DimensionalObjectProvider; import org.hisp.dhis.analytics.outlier.data.Outlier; import org.hisp.dhis.analytics.outlier.data.OutlierRequest; import org.hisp.dhis.category.CategoryOptionCombo; @@ -104,7 +104,7 @@ public class AnalyticsOutlierService { private final IdentifiableObjectManager idObjectManager; - private final DimensionalObjectProducer dimensionalObjectProducer; + private final DimensionalObjectProvider dimensionalObjectProducer; /** * Transform the incoming request into api response (json). diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/DataQueryServiceDimensionItemKeywordTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/DataQueryServiceDimensionItemKeywordTest.java index 399c9c4741fa..622f2b45975f 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/DataQueryServiceDimensionItemKeywordTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/DataQueryServiceDimensionItemKeywordTest.java @@ -122,7 +122,7 @@ class DataQueryServiceDimensionItemKeywordTest { @Mock private I18n i18n; - @InjectMocks private DimensionalObjectProducer dimensionalObjectProducer; + @InjectMocks private DimensionalObjectProvider dimensionalObjectProducer; private DefaultDataQueryService target; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/DataQueryServiceTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/DataQueryServiceTest.java index 0ea3ea985f43..11e9e08e6ee8 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/DataQueryServiceTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/DataQueryServiceTest.java @@ -52,7 +52,7 @@ @ExtendWith(MockitoExtension.class) class DataQueryServiceTest { - @Mock private DimensionalObjectProducer dimensionalObjectProducer; + @Mock private DimensionalObjectProvider dimensionalObjectProducer; @Mock private IdentifiableObjectManager idObjectManager; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/DimensionalObjectProducerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/DimensionalObjectProviderTest.java similarity index 99% rename from dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/DimensionalObjectProducerTest.java rename to dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/DimensionalObjectProviderTest.java index 3389f894a488..d0277d6514a5 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/DimensionalObjectProducerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/DimensionalObjectProviderTest.java @@ -110,12 +110,12 @@ import org.mockito.junit.jupiter.MockitoExtension; /** - * Unit tests for {@link DimensionalObjectProducer}. + * Unit tests for {@link DimensionalObjectProvider}. * * @author maikel arabori */ @ExtendWith(MockitoExtension.class) -class DimensionalObjectProducerTest { +class DimensionalObjectProviderTest { @Mock private IdentifiableObjectManager idObjectManager; @@ -135,7 +135,7 @@ class DimensionalObjectProducerTest { @Mock private I18nFormat i18nFormat; - @InjectMocks private DimensionalObjectProducer target; + @InjectMocks private DimensionalObjectProvider target; @BeforeEach public void setUp() { diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/OrgUnitTableJoinerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/OrgUnitTableJoinerTest.java index f9053ebcac55..0ad45adc592f 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/OrgUnitTableJoinerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/OrgUnitTableJoinerTest.java @@ -55,29 +55,29 @@ * @author Jim Grace */ class OrgUnitTableJoinerTest extends TestBase { - private static final OrgUnitField DEFAULT = new OrgUnitField(null); + private final OrgUnitField DEFAULT = new OrgUnitField(null); - private static final OrgUnitField ATTRIBUTE = new OrgUnitField("AttributeId"); + private final OrgUnitField ATTRIBUTE = new OrgUnitField("AttributeId"); - private static final OrgUnitField OWNER_AT_START = new OrgUnitField("OWNER_AT_START"); + private final OrgUnitField OWNER_AT_START = new OrgUnitField("OWNER_AT_START"); - private static final OrgUnitField OWNER_AT_END = new OrgUnitField("OWNER_AT_END"); + private final OrgUnitField OWNER_AT_END = new OrgUnitField("OWNER_AT_END"); - private static final Program programA = createProgram('A'); + private final Program programA = createProgram('A'); - private static final Period periodDaily = PeriodType.getPeriodFromIsoString("20230101"); + private final Period periodDaily = PeriodType.getPeriodFromIsoString("20230101"); - private static final Period periodMonthly = PeriodType.getPeriodFromIsoString("202201"); + private final Period periodMonthly = PeriodType.getPeriodFromIsoString("202201"); - private static final Period periodQuarterly = PeriodType.getPeriodFromIsoString("2022Q1"); + private final Period periodQuarterly = PeriodType.getPeriodFromIsoString("2022Q1"); - private static final Date dateA = new GregorianCalendar(2022, JANUARY, 1).getTime(); + private final Date dateA = new GregorianCalendar(2022, JANUARY, 1).getTime(); - private static final Date dateB = new GregorianCalendar(2023, JANUARY, 1).getTime(); + private final Date dateB = new GregorianCalendar(2023, JANUARY, 1).getTime(); - private static final DimensionalItemObject ouA = createOrganisationUnit('A'); + private final DimensionalItemObject ouA = createOrganisationUnit('A'); - private static final DimensionalObject ouGroupSetA = + private final DimensionalObject ouGroupSetA = new BaseDimensionalObject( "OrgUnitGrSe", DimensionType.ORGANISATION_UNIT_GROUP_SET, emptyList()); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/QueryItemHelperTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/QueryItemHelperTest.java index 1ff002351e6a..370a5198df97 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/QueryItemHelperTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/QueryItemHelperTest.java @@ -60,21 +60,21 @@ * @author Dusan Bernat */ class QueryItemHelperTest extends TestBase { - private String UID_A = CodeGenerator.generateUid(); + private final String UID_A = CodeGenerator.generateUid(); - private String UID_B = CodeGenerator.generateUid(); + private final String UID_B = CodeGenerator.generateUid(); - private String OPTION_NAME_A = "OptionA"; + private final String OPTION_NAME_A = "OptionA"; - private String OPTION_NAME_B = "OptionB"; + private final String OPTION_NAME_B = "OptionB"; - private String LEGEND_NAME_A = "LegendA"; + private final String LEGEND_NAME_A = "LegendA"; - private String LEGEND_NAME_B = "LegendB"; + private final String LEGEND_NAME_B = "LegendB"; - private String LEGEND_CODE_A = "LegendCodeA"; + private final String LEGEND_CODE_A = "LegendCodeA"; - private String LEGEND_CODE_B = "LegendCodeB"; + private final String LEGEND_CODE_B = "LegendCodeB"; @Test void testGeItemOptionValueWithIdSchemeNAME() { diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/SubexpressionPeriodOffsetUtilsTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/SubexpressionPeriodOffsetUtilsTest.java index 5ba3757e7a35..10d80dcab6ee 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/SubexpressionPeriodOffsetUtilsTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/SubexpressionPeriodOffsetUtilsTest.java @@ -55,36 +55,36 @@ */ @TestInstance(TestInstance.Lifecycle.PER_CLASS) class SubexpressionPeriodOffsetUtilsTest { - private DataElement de0 = createDataElement('A'); + private final DataElement de0 = createDataElement('A'); - private DataElement dem1 = createDataElementAWithPeriodOffset(-1); + private final DataElement dem1 = createDataElementAWithPeriodOffset(-1); - private DataElement dem2 = createDataElementAWithPeriodOffset(-2); + private final DataElement dem2 = createDataElementAWithPeriodOffset(-2); - private DataElement dep1 = createDataElementAWithPeriodOffset(1); + private final DataElement dep1 = createDataElementAWithPeriodOffset(1); - private String deAUid = de0.getUid(); + private final String deAUid = de0.getUid(); - private String expression = + private final String expression = format( "subExpression( #{%s} + #{%s}.periodOffset(-1) + #{%s}.periodOffset(-2) + #{%s}.periodOffset(1) )", deAUid, deAUid, deAUid, deAUid); - private List items = List.of(de0, dem1, dem2, dep1); + private final List items = List.of(de0, dem1, dem2, dep1); - private SubexpressionDimensionItem subex = + private final SubexpressionDimensionItem subExpr = new SubexpressionDimensionItem(expression, items, null); - private Period periodA = createPeriod("202309"); + private final Period periodA = createPeriod("202309"); - private Period periodB = createPeriod("202310"); + private final Period periodB = createPeriod("202310"); - private DataQueryParams params = + private final DataQueryParams params = DataQueryParams.newBuilder() .withPeriodType("monthly") .withPeriods(List.of(periodA, periodB)) .addDimension( - new BaseDimensionalObject(DATA_X_DIM_ID, DimensionType.DATA_X, getList(subex))) + new BaseDimensionalObject(DATA_X_DIM_ID, DimensionType.DATA_X, getList(subExpr))) .build(); @Test @@ -120,7 +120,7 @@ void testGetParamsWithDataPeriods() { getPeriodList("202307", "202308", "202309", "202310", "202311"); assertContainsOnly(expectedPeriods, result.getPeriods()); - List expectedData = List.of(subex); + List expectedData = List.of(subExpr); assertContainsOnly(expectedData, result.getAllDataDimensionItems()); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/handler/DataHandlerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/handler/DataHandlerTest.java index 1b72da84cfc6..79766d139804 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/handler/DataHandlerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/handler/DataHandlerTest.java @@ -43,12 +43,10 @@ import org.hisp.dhis.dataelement.DataElementOperand.TotalType; import org.junit.jupiter.api.Test; -/** Unit tests for {@link DataHandler}. */ class DataHandlerTest { @Test void testOperandDataQueryParamsOnlyOperands() { - // Given DataElement dataElement = new DataElement("NameA"); dataElement.setUid("uid1234567A"); dataElement.setCode("CodeA"); @@ -63,19 +61,16 @@ void testOperandDataQueryParamsOnlyOperands() { TotalType anyTotalType = COC_ONLY; - // When DataHandler dataHandler = withNullDependencies(); DataQueryParams params = dataHandler.getOperandDataQueryParams( stubParams, List.of(dataElementOperand), anyTotalType); - // Then assertEquals(1, params.getDimensions().size()); } @Test void testOperandDataQueryParamsWithCatOptionCombo() { - // Given DataElement dataElement = new DataElement("NameA"); dataElement.setUid("uid1234567A"); dataElement.setCode("CodeA"); @@ -94,18 +89,15 @@ void testOperandDataQueryParamsWithCatOptionCombo() { new BaseDimensionalObject(DATA_X_DIM_ID, DATA_X, List.of(dataElementOperand))) .build(); - // When DataHandler dataHandler = withNullDependencies(); DataQueryParams params = dataHandler.getOperandDataQueryParams(stubParams, List.of(dataElementOperand), COC_ONLY); - // Then assertEquals(2, params.getDimensions().size()); } @Test void testOperandDataQueryParamsWithCatOptionComboInFilter() { - // Given DataElement dataElement = new DataElement("NameA"); dataElement.setUid("uid1234567A"); dataElement.setCode("CodeA"); @@ -124,19 +116,16 @@ void testOperandDataQueryParamsWithCatOptionComboInFilter() { new BaseDimensionalObject(DATA_X_DIM_ID, DATA_X, List.of(dataElementOperand))) .build(); - // When DataHandler dataHandler = withNullDependencies(); DataQueryParams params = dataHandler.getOperandDataQueryParams(stubParams, List.of(dataElementOperand), COC_ONLY); - // Then assertEquals(0, params.getDimensions().size()); assertEquals(3, params.getFilters().size()); } @Test void testOperandDataQueryParamsWithAttrOptionCombo() { - // Given DataElement dataElement = new DataElement("NameA"); dataElement.setUid("uid1234567A"); dataElement.setCode("CodeA"); @@ -155,18 +144,15 @@ void testOperandDataQueryParamsWithAttrOptionCombo() { new BaseDimensionalObject(DATA_X_DIM_ID, DATA_X, List.of(dataElementOperand))) .build(); - // When DataHandler dataHandler = withNullDependencies(); DataQueryParams params = dataHandler.getOperandDataQueryParams(stubParams, List.of(dataElementOperand), AOC_ONLY); - // Then assertEquals(2, params.getDimensions().size()); } @Test void testOperandDataQueryParamsWithAttrOptionComboInFilter() { - // Given DataElement dataElement = new DataElement("NameA"); dataElement.setUid("uid1234567A"); dataElement.setCode("CodeA"); @@ -185,12 +171,10 @@ void testOperandDataQueryParamsWithAttrOptionComboInFilter() { new BaseDimensionalObject(DATA_X_DIM_ID, DATA_X, List.of(dataElementOperand))) .build(); - // When DataHandler dataHandler = withNullDependencies(); DataQueryParams params = dataHandler.getOperandDataQueryParams(stubParams, List.of(dataElementOperand), AOC_ONLY); - // Then assertEquals(0, params.getDimensions().size()); assertEquals(3, params.getFilters().size()); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/handler/SchemeIdResponseMapperTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/handler/SchemeIdResponseMapperTest.java index 2f2d8008920c..edd44fbee2e6 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/handler/SchemeIdResponseMapperTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/handler/SchemeIdResponseMapperTest.java @@ -69,7 +69,6 @@ import org.hisp.dhis.program.Program; import org.hisp.dhis.program.ProgramIndicator; import org.hisp.dhis.system.grid.ListGrid; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; @@ -79,12 +78,7 @@ */ @ExtendWith(MockitoExtension.class) class SchemeIdResponseMapperTest { - private SchemeIdResponseMapper schemeIdResponseMapper; - - @BeforeEach - public void setUp() { - schemeIdResponseMapper = new SchemeIdResponseMapper(); - } + private final SchemeIdResponseMapper schemeIdResponseMapper = new SchemeIdResponseMapper(); @Test void testGetSchemeIdResponseMapWhenOutputIdSchemeIsSetToName() { diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManagerTest.java index 5ae3c01b65d9..cfa4d4c09173 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManagerTest.java @@ -123,6 +123,7 @@ class AbstractJdbcEventAnalyticsManagerTest extends EventAnalyticsTest { @Mock private ProgramIndicatorService programIndicatorService; @Mock private ExecutionPlanStore executionPlanStore; + @Mock private OrganisationUnitService organisationUnitService; private final SqlBuilder sqlBuilder = new PostgreSqlBuilder(); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/ColumnWithNullIfAndAliasTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/ColumnWithNullIfAndAliasTest.java index 3f806ce42c14..6b0d7032775c 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/ColumnWithNullIfAndAliasTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/ColumnWithNullIfAndAliasTest.java @@ -38,12 +38,9 @@ class ColumnWithNullIfAndAliasTest { @Test void testAsSqlReturnsRightInstance() { - // given - // when ColumnAndAlias columnAndAlias = ColumnWithNullIfAndAlias.ofColumnWithNullIfAndAlias(COLUMN, ALIAS); - // then assertEquals(COLUMN, columnAndAlias.getColumn()); assertEquals(ALIAS, columnAndAlias.getAlias()); @@ -51,12 +48,9 @@ void testAsSqlReturnsRightInstance() { @Test void testAsSqlReturnsRightSqlSnippetWhenCalled() { - // given - // when ColumnAndAlias columnAndAlias = ColumnWithNullIfAndAlias.ofColumnWithNullIfAndAlias(COLUMN, ALIAS); - // then assertEquals("nullif(" + COLUMN + ",'') as \"" + ALIAS + "\"", columnAndAlias.asSql()); } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/outlier/data/OutlierQueryParserTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/outlier/data/OutlierQueryParserTest.java index e8fac6ff1c53..686074c0d7aa 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/outlier/data/OutlierQueryParserTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/outlier/data/OutlierQueryParserTest.java @@ -41,7 +41,7 @@ import java.util.List; import java.util.Set; -import org.hisp.dhis.analytics.data.DimensionalObjectProducer; +import org.hisp.dhis.analytics.data.DimensionalObjectProvider; import org.hisp.dhis.common.BaseDimensionalObject; import org.hisp.dhis.common.DisplayProperty; import org.hisp.dhis.common.IdScheme; @@ -61,7 +61,7 @@ @ExtendWith(MockitoExtension.class) class OutlierQueryParserTest { @Mock private IdentifiableObjectManager idObjectManager; - @Mock private DimensionalObjectProducer dimensionalObjectProducer; + @Mock private DimensionalObjectProvider dimensionalObjectProducer; @Mock private UserService userService; private OutlierQueryParser subject; diff --git a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/AnalyticsControllerTest.java b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/AnalyticsControllerTest.java index 7b70ecd98743..bdd79d005960 100644 --- a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/AnalyticsControllerTest.java +++ b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/AnalyticsControllerTest.java @@ -49,7 +49,7 @@ import org.hisp.dhis.analytics.DataQueryParams; import org.hisp.dhis.analytics.DataQueryService; import org.hisp.dhis.analytics.data.DefaultDataQueryService; -import org.hisp.dhis.analytics.data.DimensionalObjectProducer; +import org.hisp.dhis.analytics.data.DimensionalObjectProvider; import org.hisp.dhis.common.BaseDimensionalObject; import org.hisp.dhis.common.DimensionService; import org.hisp.dhis.common.DimensionType; @@ -90,7 +90,7 @@ class AnalyticsControllerTest { @Mock private DimensionService dimensionService; - @Mock private DimensionalObjectProducer dimensionalObjectProducer; + @Mock private DimensionalObjectProvider dimensionalObjectProducer; @Mock private DhisConfigurationProvider dhisConfigurationProvider; From 6647eacf9f88c96df8403cbf9f259ffb8be3370c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 25 Nov 2024 21:42:35 +0100 Subject: [PATCH 003/170] fix: Update code --- .../DefaultEventCoordinateServiceTest.java | 22 +++---------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/DefaultEventCoordinateServiceTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/DefaultEventCoordinateServiceTest.java index fb665fca2f99..d7bc5b74197e 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/DefaultEventCoordinateServiceTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/DefaultEventCoordinateServiceTest.java @@ -45,6 +45,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @@ -61,14 +62,13 @@ class DefaultEventCoordinateServiceTest { @Mock private TrackedEntityAttributeService attributeService; + @InjectMocks private DefaultEventCoordinateService service; + @ParameterizedTest @ValueSource(strings = {"eventgeometry", "enrollmentgeometry", "ougeometry"}) void testGetCoordinateFieldOrFail(String geometry) { when(programService.getProgram(any(String.class))).thenReturn(createProgram('A')); - EventCoordinateService service = - new DefaultEventCoordinateService(programService, dataElementService, attributeService); - assertEquals(geometry, service.getCoordinateField("A", geometry, ErrorCode.E7232)); } @@ -76,9 +76,6 @@ void testGetCoordinateFieldOrFail(String geometry) { void testGetFallbackCoordinateFieldsWithFallbackCoordinateFieldParam() { when(programService.getProgram(any(String.class))).thenReturn(createProgram('A')); - EventCoordinateService service = - new DefaultEventCoordinateService(programService, dataElementService, attributeService); - assertEquals(COL_NAME_GEOMETRY_LIST, service.getFallbackCoordinateFields("A", null, true)); } @@ -87,27 +84,18 @@ void testGetFallbackCoordinateFieldsWithFallbackCoordinateFieldParam() { void testGetFallbackCoordinateFieldsWithoutFallbackCoordinateFieldParam(String geometry) { when(programService.getProgram(any(String.class))).thenReturn(createProgram('A')); - EventCoordinateService service = - new DefaultEventCoordinateService(programService, dataElementService, attributeService); - assertEquals(List.of(geometry), service.getFallbackCoordinateFields("A", geometry, true)); } @ParameterizedTest @ValueSource(strings = {"enrollmentgeometry", "eventgeometry", "tegeometry", "ougeometry"}) void testVerifyFallbackCoordinateFieldWithRegistrationProgram(String geometry) { - EventCoordinateService service = - new DefaultEventCoordinateService(programService, dataElementService, attributeService); - assertTrue(service.isFallbackCoordinateFieldValid(true, geometry)); } @ParameterizedTest @ValueSource(strings = {"enrollmentgeometry", "eventgeometry", "ougeometry"}) void testVerifyFallbackCoordinateFieldWithoutRegistrationProgram(String geometry) { - EventCoordinateService service = - new DefaultEventCoordinateService(programService, dataElementService, attributeService); - assertTrue(service.isFallbackCoordinateFieldValid(false, geometry)); } @@ -116,12 +104,8 @@ void testVerifyFallbackCoordinateFieldWithoutRegistrationProgram(String geometry strings = {"badeventgeometry", "badenrollmentgeometry", "badtegeometry", "badougeometry"}) void testVerifyBadFallbackCoordinateField(String geometry) { when(dataElementService.getDataElement(any(String.class))).thenReturn(null); - when(attributeService.getTrackedEntityAttribute(any(String.class))).thenReturn(null); - EventCoordinateService service = - new DefaultEventCoordinateService(programService, dataElementService, attributeService); - assertThrows( IllegalQueryException.class, () -> service.isFallbackCoordinateFieldValid(false, geometry)); } From 50825c4a37a7dcd3da40d1b94209da4b9eed7dde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 25 Nov 2024 21:45:04 +0100 Subject: [PATCH 004/170] fix: Update code --- .../data/EnrollmentAggregateServiceTest.java | 22 +++++-------------- .../event/data/EventQueryServiceTest.java | 19 +++------------- .../event/data/EventQueryValidatorTest.java | 1 + 3 files changed, 9 insertions(+), 33 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EnrollmentAggregateServiceTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EnrollmentAggregateServiceTest.java index 6987fa94483c..dec8679348ac 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EnrollmentAggregateServiceTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EnrollmentAggregateServiceTest.java @@ -69,9 +69,9 @@ import org.hisp.dhis.program.ProgramStage; import org.hisp.dhis.user.SystemUser; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opengis.geometry.primitive.Point; @@ -84,8 +84,6 @@ @ExtendWith(MockitoExtension.class) class EnrollmentAggregateServiceTest { - private EnrollmentAggregateService dummyAnalyticsService; - @Mock private AnalyticsSecurityManager securityManager; @Mock private EnrollmentAnalyticsManager enrollmentAnalyticsManager; @@ -98,23 +96,13 @@ class EnrollmentAggregateServiceTest { @Mock private SchemeIdHandler schemeIdHandler; + @InjectMocks private EnrollmentAggregateService service; + @BeforeAll static void setup() { injectSecurityContextNoSettings(new SystemUser()); } - @BeforeEach - public void setUp() { - dummyAnalyticsService = - new EnrollmentAggregateService( - enrollmentAnalyticsManager, - queryPlanner, - securityManager, - queryValidator, - metadataHandler, - schemeIdHandler); - } - @Test void verifyHeaderCreationBasedOnQueryItemsAndDimensions() { // Given @@ -146,7 +134,7 @@ void verifyHeaderCreationBasedOnQueryItemsAndDimensions() { // When when(securityManager.withUserConstraints(any(EventQueryParams.class))).thenReturn(params); - Grid grid = dummyAnalyticsService.getEnrollments(params); + Grid grid = service.getEnrollments(params); // Then List headers = grid.getHeaders(); @@ -215,7 +203,7 @@ void verifyHeaderCreationBasedOnQueryItemsAndDimensionsWithSameNamesMultiStage() // When when(securityManager.withUserConstraints(any(EventQueryParams.class))).thenReturn(params); - Grid grid = dummyAnalyticsService.getEnrollments(params); + Grid grid = service.getEnrollments(params); // Then List headers = grid.getHeaders(); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EventQueryServiceTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EventQueryServiceTest.java index 0de1cb91b8cf..4d5b916e4475 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EventQueryServiceTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EventQueryServiceTest.java @@ -58,9 +58,9 @@ import org.hisp.dhis.system.database.DatabaseInfoProvider; import org.hisp.dhis.user.SystemUser; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @@ -71,8 +71,6 @@ */ @ExtendWith(MockitoExtension.class) class EventQueryServiceTest { - @Mock private EventQueryService eventQueryService; - @Mock private EventQueryValidator queryValidator; @Mock private MetadataItemsHandler metadataHandler; @@ -91,24 +89,13 @@ class EventQueryServiceTest { @Mock private SchemeIdResponseMapper schemeIdResponseMapper; + @InjectMocks private EventQueryService eventQueryService; + @BeforeAll static void setup() { injectSecurityContextNoSettings(new SystemUser()); } - @BeforeEach - public void setUp() { - eventQueryService = - new EventQueryService( - securityManager, - queryValidator, - eventAnalyticsManager, - queryPlanner, - databaseInfoProvider, - metadataHandler, - schemeIdHandler); - } - @Test void testOutputSchemeWhenSchemeIsSet() { IdScheme codeScheme = IdScheme.CODE; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EventQueryValidatorTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EventQueryValidatorTest.java index f2c27791b050..d67cb83988af 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EventQueryValidatorTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EventQueryValidatorTest.java @@ -89,6 +89,7 @@ class EventQueryValidatorTest extends TestBase { private OptionSet osA; @Mock private SystemSettingsProvider settingsProvider; + @Mock private SystemSettings settings; @Mock private QueryValidator queryValidator; From 719655c4e62838b9d41eaad939148b64bd43f659 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 25 Nov 2024 21:46:46 +0100 Subject: [PATCH 005/170] fix: Update code --- .../ProgramIndicatorSubqueryBuilderTest.java | 8 ++++---- .../event/data/queryItem/QueryItemLocatorTest.java | 14 ++------------ 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/programindicator/ProgramIndicatorSubqueryBuilderTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/programindicator/ProgramIndicatorSubqueryBuilderTest.java index 3bfee00382e5..0b8057d8ec3c 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/programindicator/ProgramIndicatorSubqueryBuilderTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/programindicator/ProgramIndicatorSubqueryBuilderTest.java @@ -47,6 +47,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @@ -61,22 +62,21 @@ class ProgramIndicatorSubqueryBuilderTest { private static final BeanRandomizer rnd = BeanRandomizer.create(); - @Mock private ProgramIndicatorService programIndicatorService; - private Program program; private Date startDate; private Date endDate; - private DefaultProgramIndicatorSubqueryBuilder subject; + @Mock private ProgramIndicatorService programIndicatorService; + + @InjectMocks private DefaultProgramIndicatorSubqueryBuilder subject; @BeforeEach public void setUp() { program = createProgram('A'); startDate = getDate(2018, 1, 1); endDate = getDate(2018, 6, 30); - subject = new DefaultProgramIndicatorSubqueryBuilder(programIndicatorService); } @Test diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/queryItem/QueryItemLocatorTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/queryItem/QueryItemLocatorTest.java index 512baf4b19ad..97f0a0837eaf 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/queryItem/QueryItemLocatorTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/queryItem/QueryItemLocatorTest.java @@ -53,7 +53,6 @@ import java.util.Set; import org.hisp.dhis.analytics.DataQueryService; import org.hisp.dhis.analytics.EventOutputType; -import org.hisp.dhis.analytics.event.QueryItemLocator; import org.hisp.dhis.analytics.event.data.DefaultQueryItemLocator; import org.hisp.dhis.common.BaseDimensionalObject; import org.hisp.dhis.common.CodeGenerator; @@ -78,6 +77,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @@ -100,7 +100,7 @@ class QueryItemLocatorTest { @Mock private RelationshipTypeService relationshipTypeService; - private QueryItemLocator subject; + @InjectMocks private DefaultQueryItemLocator subject; private Program programA; @@ -114,16 +114,6 @@ public void setUp() { dimension = CodeGenerator.generateUid(); programStageUid = CodeGenerator.generateUid(); - - subject = - new DefaultQueryItemLocator( - programStageService, - dataElementService, - attributeService, - programIndicatorService, - legendSetService, - relationshipTypeService, - dataQueryService); } @Test From c5863b157123a0322f7e98bd81ce10477a64377b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 25 Nov 2024 21:47:27 +0100 Subject: [PATCH 006/170] fix: Update code --- .../data/JdbcOrgUnitAnalyticsManagerTest.java | 19 ++++++------------- .../outlier/data/OutlierQueryParserTest.java | 3 +++ 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/orgunit/data/JdbcOrgUnitAnalyticsManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/orgunit/data/JdbcOrgUnitAnalyticsManagerTest.java index 7172035e84d5..8464921c6686 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/orgunit/data/JdbcOrgUnitAnalyticsManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/orgunit/data/JdbcOrgUnitAnalyticsManagerTest.java @@ -43,10 +43,11 @@ import org.hisp.dhis.db.sql.PostgreSqlBuilder; import org.hisp.dhis.db.sql.SqlBuilder; import org.hisp.dhis.organisationunit.OrganisationUnitGroupSet; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.Spy; import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.support.rowset.SqlRowSet; @@ -65,15 +66,9 @@ class JdbcOrgUnitAnalyticsManagerTest { @Mock private TableInfoReader tableInfoReader; - private final SqlBuilder sqlBuilder = new PostgreSqlBuilder(); + @Spy private final SqlBuilder sqlBuilder = new PostgreSqlBuilder(); - private JdbcOrgUnitAnalyticsManager jdbcOrgUnitAnalyticsManager; - - @BeforeEach - public void beforeAll() { - jdbcOrgUnitAnalyticsManager = - new JdbcOrgUnitAnalyticsManager(tableInfoReader, sqlBuilder, jdbcTemplate); - } + @InjectMocks private JdbcOrgUnitAnalyticsManager manager; @Test void testGetOrgUnitDataWithSuccess() { @@ -97,7 +92,7 @@ void testGetOrgUnitDataWithSuccess() { "analytics_rs_organisationunitgroupsetstructure", Set.of("abc123", "abc456"))) .thenReturn(Set.of()); when(jdbcTemplate.queryForRowSet(anyString())).thenReturn(sqlRowSet); - Map data = jdbcOrgUnitAnalyticsManager.getOrgUnitData(params); + Map data = manager.getOrgUnitData(params); // Then // Based on the mocked sqlRowSet. @@ -128,9 +123,7 @@ void testGetOrgUnitDataWithInvalidOrgUnitSetDimension() { // Then assertThrows( - QueryRuntimeException.class, - () -> jdbcOrgUnitAnalyticsManager.getOrgUnitData(params), - E7302.getMessage()); + QueryRuntimeException.class, () -> manager.getOrgUnitData(params), E7302.getMessage()); } private void mockSqlRowSet() { diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/outlier/data/OutlierQueryParserTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/outlier/data/OutlierQueryParserTest.java index 686074c0d7aa..de9163de50d4 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/outlier/data/OutlierQueryParserTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/outlier/data/OutlierQueryParserTest.java @@ -61,8 +61,11 @@ @ExtendWith(MockitoExtension.class) class OutlierQueryParserTest { @Mock private IdentifiableObjectManager idObjectManager; + @Mock private DimensionalObjectProvider dimensionalObjectProducer; + @Mock private UserService userService; + private OutlierQueryParser subject; @BeforeEach From d2095b99854d9566255c998e30583b2798bc90b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 25 Nov 2024 21:50:58 +0100 Subject: [PATCH 007/170] fix: Update code --- .../outlier/data/OutlierQueryParserTest.java | 5 ++--- ...nalyticsZscoreSqlStatementProcessorTest.java | 4 ---- .../CategoryOptionGroupResolverTest.java | 7 ++----- .../resolver/CategoryOptionResolverTest.java | 17 +++++------------ .../resolver/DataElementGroupResolverTest.java | 13 +++---------- 5 files changed, 12 insertions(+), 34 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/outlier/data/OutlierQueryParserTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/outlier/data/OutlierQueryParserTest.java index de9163de50d4..99118eb6e230 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/outlier/data/OutlierQueryParserTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/outlier/data/OutlierQueryParserTest.java @@ -55,6 +55,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @@ -66,7 +67,7 @@ class OutlierQueryParserTest { @Mock private UserService userService; - private OutlierQueryParser subject; + @InjectMocks private OutlierQueryParser subject; @BeforeEach void setup() { @@ -91,8 +92,6 @@ void setup() { user.setDataViewOrganisationUnits(Set.of(organisationUnit)); injectSecurityContextNoSettings(UserDetails.fromUser(user)); when(userService.getUserByUsername(anyString())).thenReturn(user); - - subject = new OutlierQueryParser(idObjectManager, dimensionalObjectProducer, userService); } @Test diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/outlier/service/AnalyticsZscoreSqlStatementProcessorTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/outlier/service/AnalyticsZscoreSqlStatementProcessorTest.java index 270d5c3f3c2b..bcc312f02f10 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/outlier/service/AnalyticsZscoreSqlStatementProcessorTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/outlier/service/AnalyticsZscoreSqlStatementProcessorTest.java @@ -57,10 +57,6 @@ class AnalyticsZscoreSqlStatementProcessorTest { private OutlierSqlStatementProcessor subject; - // ------------------------------------------------------------------------- - // Fixture - // ------------------------------------------------------------------------- - private List dataDimensions; private OrganisationUnit ouA; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/resolver/CategoryOptionGroupResolverTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/resolver/CategoryOptionGroupResolverTest.java index 71c106898d56..e0778c4cd4b5 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/resolver/CategoryOptionGroupResolverTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/resolver/CategoryOptionGroupResolverTest.java @@ -47,6 +47,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @@ -62,7 +63,7 @@ class CategoryOptionGroupResolverTest { @Mock private ExpressionService expressionService; - private ExpressionResolver resolver; + @InjectMocks private CategoryOptionGroupResolver resolver; private String uid1; @@ -89,10 +90,6 @@ public void setUp() { coc1 = createCategoryOptionCombo('X'); coc2 = createCategoryOptionCombo('Y'); coc3 = createCategoryOptionCombo('Z'); - - resolver = - new CategoryOptionGroupResolver( - expressionService, categoryOptionGroupStore, categoryOptionComboStore); } @Test diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/resolver/CategoryOptionResolverTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/resolver/CategoryOptionResolverTest.java index d5debfd3cec8..d1a62b9e4d1a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/resolver/CategoryOptionResolverTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/resolver/CategoryOptionResolverTest.java @@ -46,6 +46,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; @@ -61,7 +62,7 @@ class CategoryOptionResolverTest { @Mock private ExpressionService expressionService; - private ExpressionResolver resolver; + @InjectMocks private CategoryOptionResolver resolver; private String uid1; @@ -75,34 +76,26 @@ class CategoryOptionResolverTest { private CategoryOptionCombo coc3; - DimensionalItemId dimensionalItemId; + private DimensionalItemId dimensionalItemId; private static final String CATEGORY_OPTION_PREFIX = "co:"; @BeforeEach public void setUp() { uid1 = CodeGenerator.generateUid(); - uid2 = CodeGenerator.generateUid(); - uid3 = CodeGenerator.generateUid(); CategoryOption categoryOption = createCategoryOption('A'); coc1 = createCategoryOptionCombo('X'); - - categoryOption.addCategoryOptionCombo(coc1); - coc2 = createCategoryOptionCombo('Y'); - - categoryOption.addCategoryOptionCombo(coc2); - coc3 = createCategoryOptionCombo('Z'); + categoryOption.addCategoryOptionCombo(coc1); + categoryOption.addCategoryOptionCombo(coc2); categoryOption.addCategoryOptionCombo(coc3); - resolver = new CategoryOptionResolver(expressionService, categoryOptionStore); - when(categoryOptionStore.getByUid(anyString())).thenReturn(categoryOption); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/resolver/DataElementGroupResolverTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/resolver/DataElementGroupResolverTest.java index f5fb99dfcc85..597370b76a0a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/resolver/DataElementGroupResolverTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/resolver/DataElementGroupResolverTest.java @@ -46,6 +46,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; @@ -61,7 +62,7 @@ class DataElementGroupResolverTest { @Mock private ExpressionService expressionService; - private ExpressionResolver resolver; + @InjectMocks private DataElementGroupResolver resolver; private String uid1; @@ -75,34 +76,26 @@ class DataElementGroupResolverTest { private DataElement de3; - DimensionalItemId dimensionalItemId; + private DimensionalItemId dimensionalItemId; private static final String DATA_ELEMENT_GROUP_PREFIX = "deGroup:"; @BeforeEach public void setUp() { uid1 = CodeGenerator.generateUid(); - uid2 = CodeGenerator.generateUid(); - uid3 = CodeGenerator.generateUid(); de1 = createDataElement('X'); - de2 = createDataElement('Y'); - de3 = createDataElement('Z'); DataElementGroup dataElementGroup = createDataElementGroup('A'); dataElementGroup.addDataElement(de1); - dataElementGroup.addDataElement(de2); - dataElementGroup.addDataElement(de3); - resolver = new DataElementGroupResolver(expressionService, dataElementGroupStore); - when(dataElementGroupStore.getByUid(anyString())).thenReturn(dataElementGroup); } From 18b97820242c318b835d1e05c2795171307e064b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 25 Nov 2024 21:53:14 +0100 Subject: [PATCH 008/170] fix: Update code --- .../hisp/dhis/analytics/table/AnalyticsTableServiceTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java index 5caaa63a71ea..91389524f5de 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java @@ -55,8 +55,8 @@ */ @ExtendWith(MockitoExtension.class) class AnalyticsTableServiceTest { - @Mock private SystemSettingsProvider settingsProvider; + @Mock private SystemSettings settings; @Mock private SqlBuilder sqlBuilder; From d1520e375106fee62f9b3e4705e1f10bf2d8d275 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 25 Nov 2024 21:54:09 +0100 Subject: [PATCH 009/170] fix: Update code --- ...dbcOwnershipAnalyticsTableManagerTest.java | 34 ++++++------------- ...rackedEntityAnalyticsTableManagerTest.java | 1 - 2 files changed, 10 insertions(+), 25 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManagerTest.java index 9ec3e6847827..b399999ff371 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManagerTest.java @@ -87,8 +87,10 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.MockedStatic; +import org.mockito.Spy; import org.mockito.invocation.Invocation; import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.jdbc.core.JdbcTemplate; @@ -135,7 +137,9 @@ class JdbcOwnershipAnalyticsTableManagerTest extends TestBase { @Mock private PeriodDataProvider periodDataProvider; - private final SqlBuilder sqlBuilder = new PostgreSqlBuilder(); + @Spy private final SqlBuilder sqlBuilder = new PostgreSqlBuilder(); + + @InjectMocks private JdbcOwnershipAnalyticsTableManager manager; private static final Program programA = createProgram('A'); @@ -147,27 +151,9 @@ class JdbcOwnershipAnalyticsTableManagerTest extends TestBase { private static AnalyticsTablePartition partitionA; - private JdbcOwnershipAnalyticsTableManager target; - @BeforeEach public void setUp() { lenient().when(settingsProvider.getCurrentSettings()).thenReturn(SystemSettings.of(Map.of())); - target = - new JdbcOwnershipAnalyticsTableManager( - idObjectManager, - organisationUnitService, - categoryService, - settingsProvider, - dataApprovalLevelService, - resourceTableService, - tableHookService, - partitionManager, - databaseInfoProvider, - jdbcTemplate, - jdbcConfiguration, - analyticsTableSettings, - periodDataProvider, - sqlBuilder); tableA = new AnalyticsTable( @@ -188,7 +174,7 @@ public void setUp() { @Test void testGetAnalyticsTableType() { - assertEquals(AnalyticsTableType.OWNERSHIP, target.getAnalyticsTableType()); + assertEquals(AnalyticsTableType.OWNERSHIP, manager.getAnalyticsTableType()); } @Test @@ -197,19 +183,19 @@ void testGetAnalyticsTables() { AnalyticsTableUpdateParams params = AnalyticsTableUpdateParams.newBuilder().build(); - assertEquals(List.of(tableA, tableB), target.getAnalyticsTables(params)); + assertEquals(List.of(tableA, tableB), manager.getAnalyticsTables(params)); params = AnalyticsTableUpdateParams.newBuilder() .lastYears(AnalyticsTablePartition.LATEST_PARTITION) .build(); - assertEquals(emptyList(), target.getAnalyticsTables(params)); + assertEquals(emptyList(), manager.getAnalyticsTables(params)); } @Test void testGetPartitionChecks() { - assertTrue(target.getPartitionChecks(1, new Date()).isEmpty()); + assertTrue(manager.getPartitionChecks(1, new Date()).isEmpty()); } @Test @@ -275,7 +261,7 @@ void testPopulateTable() throws SQLException { try (MockedStatic mocked = mockStatic(JdbcOwnershipWriter.class)) { mocked.when(() -> JdbcOwnershipWriter.getInstance(any())).thenReturn(writer); - target.populateTable(params, partitionA); + manager.populateTable(params, partitionA); } List jdbcInvocations = getInvocations(jdbcTemplate); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManagerTest.java index 5a7eefacbaf4..606d62214bbe 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManagerTest.java @@ -68,7 +68,6 @@ @ExtendWith(MockitoExtension.class) class JdbcTrackedEntityAnalyticsTableManagerTest { - @Mock private JdbcTemplate jdbcTemplate; @Mock private AnalyticsTableSettings analyticsTableSettings; @Mock private PeriodDataProvider periodDataProvider; From 6c0b5a60d57a39a0a5cd1dab570fe996e3e0acd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 25 Nov 2024 21:56:36 +0100 Subject: [PATCH 010/170] fix: Update code --- .../analytics/orgunit/data/JdbcOrgUnitAnalyticsManagerTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/orgunit/data/JdbcOrgUnitAnalyticsManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/orgunit/data/JdbcOrgUnitAnalyticsManagerTest.java index 8464921c6686..27bdea994a78 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/orgunit/data/JdbcOrgUnitAnalyticsManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/orgunit/data/JdbcOrgUnitAnalyticsManagerTest.java @@ -127,9 +127,7 @@ void testGetOrgUnitDataWithInvalidOrgUnitSetDimension() { } private void mockSqlRowSet() { - // Simulate 2 results. when(sqlRowSet.next()).thenReturn(true).thenReturn(true).thenReturn(false); - when(sqlRowSet.getString("orgunit")).thenReturn("OrgUnit"); when(sqlRowSet.getString("abc123")).thenReturn("Abc123"); when(sqlRowSet.getString("abc456")).thenReturn("Abc456"); From 7d42bb0f963a1ee2b9fc93d69c440a3ffdd37a84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 25 Nov 2024 21:59:47 +0100 Subject: [PATCH 011/170] fix: Update code --- .../analytics/resolver/CategoryOptionGroupResolverTest.java | 4 ++-- .../dhis/analytics/resolver/CategoryOptionResolverTest.java | 2 +- .../dhis/analytics/resolver/DataElementGroupResolverTest.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/resolver/CategoryOptionGroupResolverTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/resolver/CategoryOptionGroupResolverTest.java index e0778c4cd4b5..ae5a6bb91c6f 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/resolver/CategoryOptionGroupResolverTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/resolver/CategoryOptionGroupResolverTest.java @@ -77,9 +77,9 @@ class CategoryOptionGroupResolverTest { private CategoryOptionCombo coc3; - DimensionalItemId dimensionalItemId; + private DimensionalItemId dimensionalItemId; - private static final String CATEGORY_OPTION_GROUP_PREFIX = "coGroup:"; + private final String CATEGORY_OPTION_GROUP_PREFIX = "coGroup:"; @BeforeEach public void setUp() { diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/resolver/CategoryOptionResolverTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/resolver/CategoryOptionResolverTest.java index d1a62b9e4d1a..245cc06733af 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/resolver/CategoryOptionResolverTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/resolver/CategoryOptionResolverTest.java @@ -78,7 +78,7 @@ class CategoryOptionResolverTest { private DimensionalItemId dimensionalItemId; - private static final String CATEGORY_OPTION_PREFIX = "co:"; + private final String CATEGORY_OPTION_PREFIX = "co:"; @BeforeEach public void setUp() { diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/resolver/DataElementGroupResolverTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/resolver/DataElementGroupResolverTest.java index 597370b76a0a..dc47bf1e99a0 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/resolver/DataElementGroupResolverTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/resolver/DataElementGroupResolverTest.java @@ -78,7 +78,7 @@ class DataElementGroupResolverTest { private DimensionalItemId dimensionalItemId; - private static final String DATA_ELEMENT_GROUP_PREFIX = "deGroup:"; + private final String DATA_ELEMENT_GROUP_PREFIX = "deGroup:"; @BeforeEach public void setUp() { From cb3d5237c77829e0769a234666bc39daca61659f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 26 Nov 2024 15:41:24 +0100 Subject: [PATCH 012/170] fix: Update code --- .../hisp/dhis/analytics/common/DimensionServiceCommonTest.java | 3 +-- .../programindicator/ProgramIndicatorSubqueryBuilderTest.java | 1 - .../dhis/analytics/table/model/AnalyticsTableColumnTest.java | 2 -- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/common/DimensionServiceCommonTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/common/DimensionServiceCommonTest.java index 1a9377fd362d..4df1d8129c22 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/common/DimensionServiceCommonTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/common/DimensionServiceCommonTest.java @@ -30,8 +30,7 @@ import java.util.function.Predicate; import org.hisp.dhis.common.ValueType; -public class DimensionServiceCommonTest { - +class DimensionServiceCommonTest { public static Predicate queryDisallowedValueTypesPredicate() { return DimensionsServiceCommon.QUERY_DISALLOWED_VALUE_TYPES::contains; } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/programindicator/ProgramIndicatorSubqueryBuilderTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/programindicator/ProgramIndicatorSubqueryBuilderTest.java index 0b8057d8ec3c..30675b3e9b83 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/programindicator/ProgramIndicatorSubqueryBuilderTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/programindicator/ProgramIndicatorSubqueryBuilderTest.java @@ -35,7 +35,6 @@ import static org.hisp.dhis.test.TestBase.createProgramIndicator; import static org.hisp.dhis.test.TestBase.getDate; import static org.mockito.Mockito.when; - import java.util.Date; import org.hisp.dhis.program.AnalyticsType; import org.hisp.dhis.program.Program; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/model/AnalyticsTableColumnTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/model/AnalyticsTableColumnTest.java index 3492631a5a36..6ea7c108a985 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/model/AnalyticsTableColumnTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/model/AnalyticsTableColumnTest.java @@ -33,7 +33,6 @@ import static org.hisp.dhis.db.model.constraint.Nullable.NULL; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; - import org.hisp.dhis.db.model.Collation; import org.junit.jupiter.api.Test; @@ -41,7 +40,6 @@ * @author Lars Helge Overland */ class AnalyticsTableColumnTest { - @Test void testIsNotNull() { AnalyticsTableColumn colA = From 9f369e42005ee717f6cda1dd43d0b99d40501868 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 26 Nov 2024 15:41:39 +0100 Subject: [PATCH 013/170] fix: Update code --- .../hisp/dhis/analytics/common/DimensionServiceCommonTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/common/DimensionServiceCommonTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/common/DimensionServiceCommonTest.java index 4df1d8129c22..1a9377fd362d 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/common/DimensionServiceCommonTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/common/DimensionServiceCommonTest.java @@ -30,7 +30,8 @@ import java.util.function.Predicate; import org.hisp.dhis.common.ValueType; -class DimensionServiceCommonTest { +public class DimensionServiceCommonTest { + public static Predicate queryDisallowedValueTypesPredicate() { return DimensionsServiceCommon.QUERY_DISALLOWED_VALUE_TYPES::contains; } From acccd5da5bfe48bc29b0232e3305f42daf736989 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 26 Nov 2024 15:43:59 +0100 Subject: [PATCH 014/170] fix: Update code --- .../table/scheduling/ContinuousAnalyticsTableJobTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/scheduling/ContinuousAnalyticsTableJobTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/scheduling/ContinuousAnalyticsTableJobTest.java index cd4c7b92641a..339100075c5e 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/scheduling/ContinuousAnalyticsTableJobTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/scheduling/ContinuousAnalyticsTableJobTest.java @@ -31,7 +31,6 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.when; - import java.util.Date; import org.hisp.dhis.analytics.AnalyticsTableGenerator; import org.hisp.dhis.analytics.common.TableInfoReader; @@ -48,6 +47,7 @@ class ContinuousAnalyticsTableJobTest { @Mock private AnalyticsTableGenerator analyticsTableGenerator; @Mock private SystemSettingsService settingsService; + @Mock private SystemSettings settings; @Mock private TableInfoReader tableInfoReader; From 97b66974df12af318762e91a80f89116bce63721 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 26 Nov 2024 15:49:37 +0100 Subject: [PATCH 015/170] fix: Update code --- .../ProgramIndicatorSubqueryBuilderTest.java | 1 + .../table/model/AnalyticsTableColumnTest.java | 1 + .../ContinuousAnalyticsTableJobTest.java | 8 +- ...rackedEntityQueryRequestValidatorTest.java | 1 - .../query/DataElementConditionTest.java | 7 - .../query/OrganisationUnitConditionTest.java | 29 --- .../query/RenderableDataValueTest.java | 1 - .../querybuilder/OffsetHelperTest.java | 6 - .../querybuilder/SqlQueryHelperTest.java | 170 +++++++++--------- 9 files changed, 91 insertions(+), 133 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/programindicator/ProgramIndicatorSubqueryBuilderTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/programindicator/ProgramIndicatorSubqueryBuilderTest.java index 30675b3e9b83..0b8057d8ec3c 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/programindicator/ProgramIndicatorSubqueryBuilderTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/programindicator/ProgramIndicatorSubqueryBuilderTest.java @@ -35,6 +35,7 @@ import static org.hisp.dhis.test.TestBase.createProgramIndicator; import static org.hisp.dhis.test.TestBase.getDate; import static org.mockito.Mockito.when; + import java.util.Date; import org.hisp.dhis.program.AnalyticsType; import org.hisp.dhis.program.Program; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/model/AnalyticsTableColumnTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/model/AnalyticsTableColumnTest.java index 6ea7c108a985..a016958cacfe 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/model/AnalyticsTableColumnTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/model/AnalyticsTableColumnTest.java @@ -33,6 +33,7 @@ import static org.hisp.dhis.db.model.constraint.Nullable.NULL; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; + import org.hisp.dhis.db.model.Collation; import org.junit.jupiter.api.Test; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/scheduling/ContinuousAnalyticsTableJobTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/scheduling/ContinuousAnalyticsTableJobTest.java index 339100075c5e..581a9a70d1bc 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/scheduling/ContinuousAnalyticsTableJobTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/scheduling/ContinuousAnalyticsTableJobTest.java @@ -31,6 +31,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.when; + import java.util.Date; import org.hisp.dhis.analytics.AnalyticsTableGenerator; import org.hisp.dhis.analytics.common.TableInfoReader; @@ -39,6 +40,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @@ -47,12 +49,12 @@ class ContinuousAnalyticsTableJobTest { @Mock private AnalyticsTableGenerator analyticsTableGenerator; @Mock private SystemSettingsService settingsService; - + @Mock private SystemSettings settings; @Mock private TableInfoReader tableInfoReader; - private ContinuousAnalyticsTableJob job; + @InjectMocks private ContinuousAnalyticsTableJob job; private final Date dateA = getDate(2024, 1, 4, 23, 0); private final Date dateB = getDate(2024, 1, 5, 2, 0); @@ -61,8 +63,6 @@ class ContinuousAnalyticsTableJobTest { @BeforeEach public void beforeEach() { when(settingsService.getCurrentSettings()).thenReturn(settings); - job = - new ContinuousAnalyticsTableJob(analyticsTableGenerator, settingsService, tableInfoReader); } @Test diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/trackedentity/TrackedEntityQueryRequestValidatorTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/trackedentity/TrackedEntityQueryRequestValidatorTest.java index 11e874343ee0..1e1b37f8629f 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/trackedentity/TrackedEntityQueryRequestValidatorTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/trackedentity/TrackedEntityQueryRequestValidatorTest.java @@ -40,7 +40,6 @@ * @author maikel arabori */ class TrackedEntityQueryRequestValidatorTest { - @Test void testValidateWhenTrackedEntityTypeIsInvalid() { String teiUid = CodeGenerator.generateUid() + "invalid"; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/trackedentity/query/DataElementConditionTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/trackedentity/query/DataElementConditionTest.java index 23d66775b561..8047be617731 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/trackedentity/query/DataElementConditionTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/trackedentity/query/DataElementConditionTest.java @@ -62,7 +62,6 @@ void testWhenNoLegendSet() { SqlParameterManager sqlParameterManager = new SqlParameterManager(); QueryContext queryContext = QueryContext.of(null, sqlParameterManager); - // SETUP DimensionParam dimensionParam = mock(DimensionParam.class); QueryItem queryItem = mock(QueryItem.class); DimensionParamItem dimensionParamItem = mock(DimensionParamItem.class); @@ -78,13 +77,11 @@ void testWhenNoLegendSet() { when(dimensionParamItem.getOperator()).thenReturn(AnalyticsQueryOperator.of(QueryOperator.EQ)); when(dimensionParamItem.getValues()).thenReturn(List.of("value")); - // CALL DataElementCondition dataElementCondition = DataElementCondition.of(queryContext, dimensionIdentifier); String rendered = dataElementCondition.render(); - // ASSERT assertEquals("(\"eventdatavalues\" -> 'uid' ->> 'value')::TEXT = :1", rendered); assertEquals("value", queryContext.getParametersPlaceHolder().get("1")); } @@ -94,7 +91,6 @@ void testWhenLegendSet() { SqlParameterManager sqlParameterManager = new SqlParameterManager(); QueryContext queryContext = QueryContext.of(null, sqlParameterManager); - // SETUP DimensionParam dimensionParam = mock(DimensionParam.class); QueryItem queryItem = mock(QueryItem.class); DimensionParamItem dimensionParamItem = mock(DimensionParamItem.class); @@ -105,7 +101,6 @@ void testWhenLegendSet() { when(dimensionParam.getUid()).thenReturn("uid"); when(queryItem.hasLegendSet()).thenReturn(true); - // in the assertion we will check that the type is force to DECIMAL when(dimensionParam.getValueType()).thenReturn(ValueType.TEXT); when(dimensionParam.getItems()).thenReturn(List.of(dimensionParamItem)); @@ -121,13 +116,11 @@ void testWhenLegendSet() { when(legend.getStartValue()).thenReturn(1.0); when(legend.getEndValue()).thenReturn(2.0); - // CALL DataElementCondition dataElementCondition = DataElementCondition.of(queryContext, dimensionIdentifier); String rendered = dataElementCondition.render(); - // ASSERT assertEquals( "(\"eventdatavalues\" -> 'uid' ->> 'value')::DECIMAL >= :1 " + "and (\"eventdatavalues\" -> 'uid' ->> 'value')::DECIMAL < :2", diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/trackedentity/query/OrganisationUnitConditionTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/trackedentity/query/OrganisationUnitConditionTest.java index 7fac1fb4b9e9..606f35a0cc5a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/trackedentity/query/OrganisationUnitConditionTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/trackedentity/query/OrganisationUnitConditionTest.java @@ -60,11 +60,9 @@ import org.hisp.dhis.trackedentity.TrackedEntityType; import org.junit.jupiter.api.Test; -/** Unit tests for {@link OrganisationUnitCondition}. */ class OrganisationUnitConditionTest { @Test void testTeiOuMultipleOusProduceCorrectSql() { - // Given List ous = List.of("ou1", "ou2"); DimensionIdentifier dimensionIdentifier = stubDimensionIdentifier(ous, null, null); @@ -82,17 +80,14 @@ void testTeiOuMultipleOusProduceCorrectSql() { OrganisationUnitCondition organisationUnitCondition = OrganisationUnitCondition.of(dimensionIdentifier, queryContext); - // When String render = organisationUnitCondition.render(); - // Then assertEquals("t_1.\"ou\" in (:1)", render); assertEquals(ous, queryContext.getParametersPlaceHolder().get("1")); } @Test void testChildrenOuModeProduceCorrectSql() { - // Given List ous = List.of("ou1", "ou2"); DimensionIdentifier dimensionIdentifier = @@ -125,13 +120,11 @@ void testChildrenOuModeProduceCorrectSql() { OrganisationUnitCondition organisationUnitCondition = OrganisationUnitCondition.of(dimensionIdentifier, queryContext); - // When String render = organisationUnitCondition.render(); List expected = ous.stream().flatMap(ouId -> Stream.of(ouId + "_children1", ouId + "_children2")).toList(); - // Then assertEquals("t_1.\"ou\" in (:1)", render); assertTrue( isEqualCollection( @@ -140,7 +133,6 @@ void testChildrenOuModeProduceCorrectSql() { @Test void testDescendantOuModeProduceCorrectSql() { - // Given List ous = List.of("ou1", "ou2"); DimensionIdentifier dimensionIdentifier = @@ -155,7 +147,6 @@ void testDescendantOuModeProduceCorrectSql() { return organisationUnit; }); - // Descendant is the default ouMode. CommonRequestParams requestParams = new CommonRequestParams(); ContextParams contextParams = @@ -169,16 +160,13 @@ void testDescendantOuModeProduceCorrectSql() { OrganisationUnitCondition organisationUnitCondition = OrganisationUnitCondition.of(dimensionIdentifier, queryContext); - // When String render = organisationUnitCondition.render(); - // Then assertEquals("(t_1.\"uidlevel1\" = :1 or t_1.\"uidlevel1\" = :2)", render); } @Test void testEmptyOuProduceFalse() { - // Given List ous = List.of(); DimensionIdentifier dimensionIdentifier = stubDimensionIdentifier(ous, null, null); @@ -197,17 +185,14 @@ void testEmptyOuProduceFalse() { OrganisationUnitCondition organisationUnitCondition = OrganisationUnitCondition.of(dimensionIdentifier, queryContext); - // When String render = organisationUnitCondition.render(); - // Then assertEquals("false", render); assertTrue(queryContext.getParametersPlaceHolder().isEmpty()); } @Test void testTeiOuSingleOusProduceCorrectSql() { - // Given List ous = List.of("ou1"); DimensionIdentifier dimensionIdentifier = @@ -227,17 +212,14 @@ void testTeiOuSingleOusProduceCorrectSql() { OrganisationUnitCondition organisationUnitCondition = OrganisationUnitCondition.of(dimensionIdentifier, queryContext); - // When String render = organisationUnitCondition.render(); - // Then assertEquals("t_1.\"ou\" = :1", render); assertEquals(ous.get(0), queryContext.getParametersPlaceHolder().get("1")); } @Test void testEventOuSingleOusProduceCorrectSql() { - // Given List ous = List.of("ou1"); DimensionIdentifier dimensionIdentifier = @@ -257,17 +239,14 @@ void testEventOuSingleOusProduceCorrectSql() { OrganisationUnitCondition organisationUnitCondition = OrganisationUnitCondition.of(dimensionIdentifier, queryContext); - // When String statement = organisationUnitCondition.render(); - // Then assertEquals("\"Z8z5uu61HAb.tO8L1aBitDm\".\"ou\" = :1", statement); assertEquals(ous.get(0), queryContext.getParametersPlaceHolder().get("1")); } @Test void testEventOuMultipleOusProduceCorrectSql() { - // Given List ous = List.of("ou1", "ou2"); DimensionIdentifier dimensionIdentifier = @@ -291,17 +270,14 @@ void testEventOuMultipleOusProduceCorrectSql() { OrganisationUnitCondition organisationUnitCondition = OrganisationUnitCondition.of(dimensionIdentifier, queryContext); - // When String statement = organisationUnitCondition.render(); - // Then assertEquals("\"Z8z5uu61HAb.tO8L1aBitDm\".\"ou\" in (:1)", statement); assertEquals(ous, queryContext.getParametersPlaceHolder().get("1")); } @Test void testEnrollmentOuSingleOusProduceCorrectSql() { - // Given List ous = List.of("ou1"); DimensionIdentifier dimensionIdentifier = @@ -325,17 +301,14 @@ void testEnrollmentOuSingleOusProduceCorrectSql() { OrganisationUnitCondition organisationUnitCondition = OrganisationUnitCondition.of(dimensionIdentifier, queryContext); - // When String statement = organisationUnitCondition.render(); - // Then assertEquals("\"Z8z5uu61HAb\".\"ou\" = :1", statement); assertEquals(ous.get(0), queryContext.getParametersPlaceHolder().get("1")); } @Test void testEnrollmentOuMultipleOusProduceCorrectSql() { - // Given List ous = List.of("ou1", "ou2"); DimensionIdentifier dimensionIdentifier = @@ -358,10 +331,8 @@ void testEnrollmentOuMultipleOusProduceCorrectSql() { OrganisationUnitCondition organisationUnitCondition = OrganisationUnitCondition.of(dimensionIdentifier, queryContext); - // When String statement = organisationUnitCondition.render(); - // Then assertEquals("\"Z8z5uu61HAb\".\"ou\" in (:1)", statement); assertEquals(ous, queryContext.getParametersPlaceHolder().get("1")); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/trackedentity/query/RenderableDataValueTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/trackedentity/query/RenderableDataValueTest.java index 547005a9ac66..edfe1644c305 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/trackedentity/query/RenderableDataValueTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/trackedentity/query/RenderableDataValueTest.java @@ -39,7 +39,6 @@ import org.junit.jupiter.api.Test; class RenderableDataValueTest { - @Test void testRender() { RenderableDataValue renderableDataValue = diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/trackedentity/query/context/querybuilder/OffsetHelperTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/trackedentity/query/context/querybuilder/OffsetHelperTest.java index ba1176e86bd4..5ddfb28318ea 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/trackedentity/query/context/querybuilder/OffsetHelperTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/trackedentity/query/context/querybuilder/OffsetHelperTest.java @@ -35,21 +35,17 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; -/** Tests for {@link OffsetHelper}. */ class OffsetHelperTest { @ParameterizedTest(name = "testGetItemBasedOnOffset - {index}") @CsvSource({"2,d", "1,e", "0,a", "-1,b", "-2,c"}) void testGetItemBasedOnOffset(String offsetParam, String expectedResponse) { - // Given Stream stream = Stream.of("a", "b", "c", "d", "e"); Comparator comparator = Comparator.naturalOrder(); int offset = Integer.parseInt(offsetParam); - // When Optional result = OffsetHelper.getItemBasedOnOffset(stream, comparator, offset); - // Then Assertions.assertTrue(result.isPresent()); Assertions.assertEquals(expectedResponse, result.get()); } @@ -57,10 +53,8 @@ void testGetItemBasedOnOffset(String offsetParam, String expectedResponse) { @ParameterizedTest @CsvSource({"1,1,asc", "2,2,asc", "0,1,desc", "-1,2,desc", "-2,3,desc"}) void testGetOffset(String offsetParam, String expectedOffset, String expectedDirection) { - // When Offset offset = OffsetHelper.getOffset(Integer.parseInt(offsetParam)); - // Then Assertions.assertEquals(expectedOffset, offset.offset()); Assertions.assertEquals(expectedDirection, offset.direction()); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/trackedentity/query/context/querybuilder/SqlQueryHelperTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/trackedentity/query/context/querybuilder/SqlQueryHelperTest.java index 289140c447d3..286a29c25fee 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/trackedentity/query/context/querybuilder/SqlQueryHelperTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/trackedentity/query/context/querybuilder/SqlQueryHelperTest.java @@ -102,26 +102,26 @@ void test_subQuery_enrollment() { assertEquals( """ - (select field - from (select *, - row_number() over ( partition by trackedentity - order by enrollmentdate desc ) as rn - from analytics_te_enrollment_trackedentitytype - where program = 'programUid' - and t_1.trackedentity = trackedentity) en - where en.rn = 1)""", + (select field + from (select *, + row_number() over ( partition by trackedentity + order by enrollmentdate desc ) as rn + from analytics_te_enrollment_trackedentitytype + where program = 'programUid' + and t_1.trackedentity = trackedentity) en + where en.rn = 1)""", SqlQueryHelper.buildOrderSubQuery(testedDimension, () -> "field").render()); assertEquals( """ - exists(select 1 - from (select * - from (select *, row_number() over (partition by trackedentity order by enrollmentdate desc) as rn - from analytics_te_enrollment_trackedentitytype - where program = 'programUid' - and trackedentity = t_1.trackedentity) en - where en.rn = 1) as "prefix" - where field)""", + exists(select 1 + from (select * + from (select *, row_number() over (partition by trackedentity order by enrollmentdate desc) as rn + from analytics_te_enrollment_trackedentitytype + where program = 'programUid' + and trackedentity = t_1.trackedentity) en + where en.rn = 1) as "prefix" + where field)""", SqlQueryHelper.buildExistsValueSubquery(testedDimension, () -> "field").render()); } @@ -148,42 +148,42 @@ void test_subQuery_event() { assertEquals( """ - (select field - from (select *, - row_number() over ( partition by enrollment - order by occurreddate desc ) as rn - from analytics_te_event_trackedentitytype events - where programstage = 'programStageUid' - and enrollment = (select enrollment - from (select *, - row_number() over ( partition by trackedentity - order by enrollmentdate desc ) as rn - from analytics_te_enrollment_trackedentitytype - where program = 'programUid' - and t_1.trackedentity = trackedentity) en - where en.rn = 1) - and status != 'SCHEDULE') ev - where ev.rn = 1)""", + (select field + from (select *, + row_number() over ( partition by enrollment + order by occurreddate desc ) as rn + from analytics_te_event_trackedentitytype events + where programstage = 'programStageUid' + and enrollment = (select enrollment + from (select *, + row_number() over ( partition by trackedentity + order by enrollmentdate desc ) as rn + from analytics_te_enrollment_trackedentitytype + where program = 'programUid' + and t_1.trackedentity = trackedentity) en + where en.rn = 1) + and status != 'SCHEDULE') ev + where ev.rn = 1)""", SqlQueryHelper.buildOrderSubQuery(testedDimension, () -> "field").render()); assertEquals( """ - exists(select 1 - from (select * - from (select *, row_number() over (partition by trackedentity order by enrollmentdate desc) as rn - from analytics_te_enrollment_trackedentitytype - where program = 'programUid' - and trackedentity = t_1.trackedentity) en - where en.rn = 1) as "enrollmentSubqueryAlias" - where exists(select 1 - from (select * - from (select *, row_number() over ( partition by enrollment order by occurreddate desc ) as rn - from analytics_te_event_trackedentitytype - where "enrollmentSubqueryAlias".enrollment = enrollment - and programstage = 'programStageUid' - and status != 'SCHEDULE') ev - where ev.rn = 1) as "prefix" - where field))""", + exists(select 1 + from (select * + from (select *, row_number() over (partition by trackedentity order by enrollmentdate desc) as rn + from analytics_te_enrollment_trackedentitytype + where program = 'programUid' + and trackedentity = t_1.trackedentity) en + where en.rn = 1) as "enrollmentSubqueryAlias" + where exists(select 1 + from (select * + from (select *, row_number() over ( partition by enrollment order by occurreddate desc ) as rn + from analytics_te_event_trackedentitytype + where "enrollmentSubqueryAlias".enrollment = enrollment + and programstage = 'programStageUid' + and status != 'SCHEDULE') ev + where ev.rn = 1) as "prefix" + where field))""", SqlQueryHelper.buildExistsValueSubquery(testedDimension, () -> "field").render()); } @@ -211,47 +211,47 @@ void test_subQuery_data_element() { assertEquals( """ - (select field - from analytics_te_event_trackedentitytype - where event = (select event - from (select *, - row_number() over ( partition by enrollment - order by occurreddate desc ) as rn - from analytics_te_event_trackedentitytype events - where programstage = 'programStageUid' - and enrollment = (select enrollment - from (select *, - row_number() over ( partition by trackedentity - order by enrollmentdate desc ) as rn - from analytics_te_enrollment_trackedentitytype - where program = 'programUid' - and t_1.trackedentity = trackedentity) en - where en.rn = 1) - and status != 'SCHEDULE') ev - where ev.rn = 1))""", + (select field + from analytics_te_event_trackedentitytype + where event = (select event + from (select *, + row_number() over ( partition by enrollment + order by occurreddate desc ) as rn + from analytics_te_event_trackedentitytype events + where programstage = 'programStageUid' + and enrollment = (select enrollment + from (select *, + row_number() over ( partition by trackedentity + order by enrollmentdate desc ) as rn + from analytics_te_enrollment_trackedentitytype + where program = 'programUid' + and t_1.trackedentity = trackedentity) en + where en.rn = 1) + and status != 'SCHEDULE') ev + where ev.rn = 1))""", SqlQueryHelper.buildOrderSubQuery(testedDimension, () -> "field").render()); assertEquals( """ - exists(select 1 - from (select * - from (select *, row_number() over (partition by trackedentity order by enrollmentdate desc) as rn - from analytics_te_enrollment_trackedentitytype - where program = 'programUid' - and trackedentity = t_1.trackedentity) en - where en.rn = 1) as "enrollmentSubqueryAlias" - where exists(select 1 - from (select * - from (select *, row_number() over ( partition by enrollment order by occurreddate desc ) as rn - from analytics_te_event_trackedentitytype - where "enrollmentSubqueryAlias".enrollment = enrollment - and programstage = 'programStageUid' - and status != 'SCHEDULE') ev - where ev.rn = 1) as "prefix" - where exists(select 1 - from analytics_te_event_trackedentitytype - where "prefix".event = event - and field)))""", + exists(select 1 + from (select * + from (select *, row_number() over (partition by trackedentity order by enrollmentdate desc) as rn + from analytics_te_enrollment_trackedentitytype + where program = 'programUid' + and trackedentity = t_1.trackedentity) en + where en.rn = 1) as "enrollmentSubqueryAlias" + where exists(select 1 + from (select * + from (select *, row_number() over ( partition by enrollment order by occurreddate desc ) as rn + from analytics_te_event_trackedentitytype + where "enrollmentSubqueryAlias".enrollment = enrollment + and programstage = 'programStageUid' + and status != 'SCHEDULE') ev + where ev.rn = 1) as "prefix" + where exists(select 1 + from analytics_te_event_trackedentitytype + where "prefix".event = event + and field)))""", SqlQueryHelper.buildExistsValueSubquery(testedDimension, () -> "field").render()); } From 7c725c76e0e71f1b49f766a64dc58667a8c3388c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 26 Nov 2024 15:50:55 +0100 Subject: [PATCH 016/170] fix: Update code --- .../context/sql/SqlQueryCreatorServiceTest.java | 12 ------------ .../dhis/analytics/util/AnalyticsColumnAsserter.java | 1 - .../dhis/common/BaseDimensionalItemObjectTest.java | 6 ------ .../resourcetable/table/UniqueNameContextTest.java | 1 - 4 files changed, 20 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/trackedentity/query/context/sql/SqlQueryCreatorServiceTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/trackedentity/query/context/sql/SqlQueryCreatorServiceTest.java index ad9781863681..5be9d61daf2a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/trackedentity/query/context/sql/SqlQueryCreatorServiceTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/trackedentity/query/context/sql/SqlQueryCreatorServiceTest.java @@ -93,7 +93,6 @@ void setUp() { @Test void testSqlQueryRenderingWithOrgUnitNameObject() { - // given TrackedEntityQueryParams trackedEntityQueryParams = TrackedEntityQueryParams.builder().trackedEntityType(createTrackedEntityType('A')).build(); @@ -104,18 +103,15 @@ void testSqlQueryRenderingWithOrgUnitNameObject() { .commonParsed(stubSortingCommonParams(null, 1, "ouname")) .build(); - // when String sql = sqlQueryCreatorService.getSqlQueryCreator(contextParams).createForSelect().getStatement(); - // then assertTrue(sql.contains("ouname")); assertContains("order by t_1.\"ouname\" desc", sql); } @Test void testSqlQueryRenderingWithCommonDimensionalObject() { - // when DimensionalObject dimensionalObject = new BaseDimensionalObject("abc"); TrackedEntityType trackedEntityType = createTrackedEntityType('A'); @@ -132,17 +128,14 @@ void testSqlQueryRenderingWithCommonDimensionalObject() { .commonParsed(stubSortingCommonParams(program, 1, dimensionalObject)) .build(); - // when String sql = sqlQueryCreatorService.getSqlQueryCreator(contextParams).createForSelect().getStatement(); - // then assertTrue(sql.contains(" order by (select (\"eventdatavalues\" -> 'abc' ->> 'value')::TEXT")); } @Test void testEnrolledInProgramWhenSpecifiedInRequest() { - // given CommonParsedParams commonParsed = CommonParsedParams.builder() .programs(List.of(mockProgram("program1"), mockProgram("program2"))) @@ -162,18 +155,15 @@ void testEnrolledInProgramWhenSpecifiedInRequest() { .commonParsed(commonParsed) .build(); - // when String sql = sqlQueryCreatorService.getSqlQueryCreator(contextParams).createForSelect().getStatement(); - // then assertTrue(sql.contains("ouname")); assertContains("t_1.\"program1\" and t_1.\"program2\"", sql); } @Test void testEnrolledInProgramWhenNotSpecifiedInRequest() { - // given CommonParsedParams commonParsed = CommonParsedParams.builder() .programs(List.of(mockProgram("program1"), mockProgram("program2"))) @@ -191,11 +181,9 @@ void testEnrolledInProgramWhenNotSpecifiedInRequest() { .commonRaw(requestParams) .build(); - // when String sql = sqlQueryCreatorService.getSqlQueryCreator(contextParams).createForSelect().getStatement(); - // then assertTrue(sql.contains("ouname")); assertContains("(t_1.\"program1\" or t_1.\"program2\")", sql); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/util/AnalyticsColumnAsserter.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/util/AnalyticsColumnAsserter.java index ef6e06592070..2c9278516e70 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/util/AnalyticsColumnAsserter.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/util/AnalyticsColumnAsserter.java @@ -39,7 +39,6 @@ */ @Setter public class AnalyticsColumnAsserter { - /** The analytics table column to verify. */ private AnalyticsTableColumn actual; public void verify(AnalyticsTableColumn expected) { diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/common/BaseDimensionalItemObjectTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/common/BaseDimensionalItemObjectTest.java index 3259349deebe..57912122b542 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/common/BaseDimensionalItemObjectTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/common/BaseDimensionalItemObjectTest.java @@ -35,25 +35,19 @@ class BaseDimensionalItemObjectTest { @Test void testWhenBaseDimensionalItemObjectAggregationTypeIsNoneTotalAggregationTypeIsNone() { - // given BaseDimensionalItemObject baseDimensionalItemObject = new BaseDimensionalItemObject(); - // when baseDimensionalItemObject.setAggregationType(AggregationType.NONE); - // then assertSame(TotalAggregationType.NONE, baseDimensionalItemObject.getTotalAggregationType()); } @Test void testWhenBaseDimensionalItemObjectAggregationTypeIsNotNoneTotalAggregationTypeIsSum() { - // given BaseDimensionalItemObject baseDimensionalItemObject = new BaseDimensionalItemObject(); - // when baseDimensionalItemObject.setAggregationType(AggregationType.AVERAGE_SUM_ORG_UNIT); - // then assertSame(TotalAggregationType.SUM, baseDimensionalItemObject.getTotalAggregationType()); } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/resourcetable/table/UniqueNameContextTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/resourcetable/table/UniqueNameContextTest.java index 0e1bc425adc6..380e698dc9ed 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/resourcetable/table/UniqueNameContextTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/resourcetable/table/UniqueNameContextTest.java @@ -38,7 +38,6 @@ * @author Jan Bernitt */ class UniqueNameContextTest { - private final UniqueNameContext context = new UniqueNameContext(); @Test From 490ea090b88a393e2acd2789b4146e83217c4823 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 26 Nov 2024 16:18:38 +0100 Subject: [PATCH 017/170] fix: Update code --- .../java/org/hisp/dhis/common/DimensionalObjectUtils.java | 8 +++----- .../java/org/hisp/dhis/analytics/common/ColumnHeader.java | 3 +-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObjectUtils.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObjectUtils.java index f0d2ceae6cc3..f4a293029fea 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObjectUtils.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObjectUtils.java @@ -38,9 +38,6 @@ import static org.hisp.dhis.common.DimensionalObject.ITEM_SEP; import static org.hisp.dhis.common.DimensionalObject.OPTION_SEP; import static org.hisp.dhis.expression.ExpressionService.SYMBOL_WILDCARD; - -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -53,7 +50,6 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; -import lombok.NoArgsConstructor; import org.apache.commons.collections4.ListUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Triple; @@ -63,6 +59,9 @@ import org.hisp.dhis.eventvisualization.EventRepetition; import org.hisp.dhis.program.Program; import org.hisp.dhis.program.ProgramStage; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import lombok.NoArgsConstructor; /** * @author Lars Helge Overland @@ -141,7 +140,6 @@ public static DimensionalObject linkAssociations( EventAnalyticalObject eventAnalyticalObject, DimensionalObject dimensionalObject, Attribute parent) { - // Associating event repetitions. List repetitions = eventAnalyticalObject.getEventRepetitions(); if (isNotEmpty(repetitions)) { diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/analytics/common/ColumnHeader.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/analytics/common/ColumnHeader.java index 814be376f4db..1e9e65e1681a 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/analytics/common/ColumnHeader.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/analytics/common/ColumnHeader.java @@ -34,9 +34,8 @@ public enum ColumnHeader { GEOMETRY("geometry", "Geometry"), ENROLLMENT_DATE("enrollmentdate", "Enrollment date"), /** - * @deprecated use @Link{#OCCURRED_DATE} instead. + * Deprecated since 2.42. Use @Link{#OCCURRED_DATE} instead. */ - @Deprecated(since = "2.42") INCIDENT_DATE("incidentdate", "Incident date"), OCCURRED_DATE("occurreddate", "Occurred date"), STORED_BY("storedby", "Stored by"), From 781d2e462586495300d7e75c431eb4ab9a7f23b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 26 Nov 2024 16:33:27 +0100 Subject: [PATCH 018/170] fix: Update code --- .../table/AbstractEventJdbcTableManager.java | 6 +++--- .../table/AbstractJdbcTableManager.java | 14 +++++++------- .../table/JdbcEventAnalyticsTableManager.java | 16 ++++++++-------- ...lumnType.java => AnalyticsDimensionType.java} | 2 +- .../table/model/AnalyticsTableColumn.java | 7 ++++--- .../analytics/util/AnalyticsIndexHelper.java | 4 ++-- 6 files changed, 25 insertions(+), 24 deletions(-) rename dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/model/{AnalyticsColumnType.java => AnalyticsDimensionType.java} (97%) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 0b07dced4214..225991dcba23 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -38,7 +38,7 @@ import java.util.Map; import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.partition.PartitionManager; -import org.hisp.dhis.analytics.table.model.AnalyticsColumnType; +import org.hisp.dhis.analytics.table.model.AnalyticsDimensionType; import org.hisp.dhis.analytics.table.model.AnalyticsTableColumn; import org.hisp.dhis.analytics.table.model.AnalyticsTablePartition; import org.hisp.dhis.analytics.table.model.Skip; @@ -259,7 +259,7 @@ protected List getTrackedEntityAttributeColumns(Program pr columns.add( AnalyticsTableColumn.builder() .name(attribute.getUid()) - .columnType(AnalyticsColumnType.DYNAMIC) + .dimensionType(AnalyticsDimensionType.DYNAMIC) .dataType(dataType) .selectExpression(sql) .skipIndex(skipIndex) @@ -272,7 +272,7 @@ protected List getTrackedEntityAttributeColumns(Program pr columns.add( AnalyticsTableColumn.builder() .name((attribute.getUid() + OU_NAME_COL_SUFFIX)) - .columnType(AnalyticsColumnType.DYNAMIC) + .dimensionType(AnalyticsDimensionType.DYNAMIC) .dataType(TEXT) .selectExpression(ouNameSql) .skipIndex(SKIP) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java index 1d16314a85cf..94a49a3ea16a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java @@ -54,7 +54,7 @@ import org.hisp.dhis.analytics.AnalyticsTableType; import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; import org.hisp.dhis.analytics.partition.PartitionManager; -import org.hisp.dhis.analytics.table.model.AnalyticsColumnType; +import org.hisp.dhis.analytics.table.model.AnalyticsDimensionType; import org.hisp.dhis.analytics.table.model.AnalyticsTable; import org.hisp.dhis.analytics.table.model.AnalyticsTableColumn; import org.hisp.dhis.analytics.table.model.AnalyticsTablePartition; @@ -537,7 +537,7 @@ protected List getOrganisationUnitGroupSetColumns() { String name = ougs.getUid(); return AnalyticsTableColumn.builder() .name(name) - .columnType(AnalyticsColumnType.DYNAMIC) + .dimensionType(AnalyticsDimensionType.DYNAMIC) .dataType(CHARACTER_11) .selectExpression("ougs." + quote(name)) .skipIndex(skipIndex(ougs)) @@ -554,7 +554,7 @@ protected List getDataElementGroupSetColumns() { String name = degs.getUid(); return AnalyticsTableColumn.builder() .name(name) - .columnType(AnalyticsColumnType.DYNAMIC) + .dimensionType(AnalyticsDimensionType.DYNAMIC) .dataType(CHARACTER_11) .selectExpression("degs." + quote(name)) .skipIndex(skipIndex(degs)) @@ -571,7 +571,7 @@ protected List getDisaggregationCategoryOptionGroupSetColu String name = cogs.getUid(); return AnalyticsTableColumn.builder() .name(name) - .columnType(AnalyticsColumnType.DYNAMIC) + .dimensionType(AnalyticsDimensionType.DYNAMIC) .dataType(CHARACTER_11) .selectExpression("dcs." + quote(name)) .skipIndex(skipIndex(cogs)) @@ -588,7 +588,7 @@ protected List getAttributeCategoryOptionGroupSetColumns() String name = cogs.getUid(); return AnalyticsTableColumn.builder() .name(name) - .columnType(AnalyticsColumnType.DYNAMIC) + .dimensionType(AnalyticsDimensionType.DYNAMIC) .dataType(CHARACTER_11) .selectExpression("acs." + quote(name)) .skipIndex(skipIndex(cogs)) @@ -605,7 +605,7 @@ protected List getDisaggregationCategoryColumns() { String name = category.getUid(); return AnalyticsTableColumn.builder() .name(name) - .columnType(AnalyticsColumnType.DYNAMIC) + .dimensionType(AnalyticsDimensionType.DYNAMIC) .dataType(CHARACTER_11) .selectExpression("dcs." + quote(name)) .skipIndex(skipIndex(category)) @@ -622,7 +622,7 @@ protected List getAttributeCategoryColumns() { String name = category.getUid(); return AnalyticsTableColumn.builder() .name(name) - .columnType(AnalyticsColumnType.DYNAMIC) + .dimensionType(AnalyticsDimensionType.DYNAMIC) .dataType(CHARACTER_11) .selectExpression("acs." + quote(name)) .skipIndex(skipIndex(category)) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 077b1284393a..1016ca4f3dcf 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -55,7 +55,7 @@ import org.hisp.dhis.analytics.AnalyticsTableType; import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; import org.hisp.dhis.analytics.partition.PartitionManager; -import org.hisp.dhis.analytics.table.model.AnalyticsColumnType; +import org.hisp.dhis.analytics.table.model.AnalyticsDimensionType; import org.hisp.dhis.analytics.table.model.AnalyticsTable; import org.hisp.dhis.analytics.table.model.AnalyticsTableColumn; import org.hisp.dhis.analytics.table.model.AnalyticsTablePartition; @@ -407,7 +407,7 @@ private List getColumns(Program program) { columns.add( AnalyticsTableColumn.builder() .name(category.getUid()) - .columnType(AnalyticsColumnType.DYNAMIC) + .dimensionType(AnalyticsDimensionType.DYNAMIC) .dataType(CHARACTER_11) .selectExpression("acs." + quote(category.getUid())) .created(category.getCreated()) @@ -496,7 +496,7 @@ private List getColumnForDataElement( columns.add( AnalyticsTableColumn.builder() .name(dataElement.getUid()) - .columnType(AnalyticsColumnType.DYNAMIC) + .dimensionType(AnalyticsDimensionType.DYNAMIC) .dataType(dataType) .selectExpression(sql) .skipIndex(skipIndex) @@ -523,7 +523,7 @@ private List getColumnForOrgUnitDataElement( columns.add( AnalyticsTableColumn.builder() .name((dataElement.getUid() + OU_GEOMETRY_COL_SUFFIX)) - .columnType(AnalyticsColumnType.DYNAMIC) + .dimensionType(AnalyticsDimensionType.DYNAMIC) .dataType(GEOMETRY) .selectExpression(geoSql) .indexType(IndexType.GIST) @@ -536,7 +536,7 @@ private List getColumnForOrgUnitDataElement( columns.add( AnalyticsTableColumn.builder() .name((dataElement.getUid() + OU_NAME_COL_SUFFIX)) - .columnType(AnalyticsColumnType.DYNAMIC) + .dimensionType(AnalyticsDimensionType.DYNAMIC) .dataType(TEXT) .selectExpression(ouNameSql) .skipIndex(SKIP) @@ -562,7 +562,7 @@ private List getColumnForTrackedEntityAttribute( columns.add( AnalyticsTableColumn.builder() .name(attribute.getUid()) - .columnType(AnalyticsColumnType.DYNAMIC) + .dimensionType(AnalyticsDimensionType.DYNAMIC) .dataType(dataType) .selectExpression(sql) .skipIndex(skipIndex) @@ -633,7 +633,7 @@ private List getColumnsForOrgUnitTrackedEntityAttribute( columns.add( AnalyticsTableColumn.builder() .name((attribute.getUid() + OU_GEOMETRY_COL_SUFFIX)) - .columnType(AnalyticsColumnType.DYNAMIC) + .dimensionType(AnalyticsDimensionType.DYNAMIC) .dataType(GEOMETRY) .selectExpression(geoSql) .indexType(IndexType.GIST) @@ -646,7 +646,7 @@ private List getColumnsForOrgUnitTrackedEntityAttribute( columns.add( AnalyticsTableColumn.builder() .name((attribute.getUid() + OU_NAME_COL_SUFFIX)) - .columnType(AnalyticsColumnType.DYNAMIC) + .dimensionType(AnalyticsDimensionType.DYNAMIC) .dataType(TEXT) .selectExpression(ouNameSql) .skipIndex(SKIP) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/model/AnalyticsColumnType.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/model/AnalyticsDimensionType.java similarity index 97% rename from dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/model/AnalyticsColumnType.java rename to dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/model/AnalyticsDimensionType.java index e00267ed3717..cd54f16e4e60 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/model/AnalyticsColumnType.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/model/AnalyticsDimensionType.java @@ -31,7 +31,7 @@ * Represents a type of dimension, either static, meaning fixed, or dynamic, meaning based on a * dimensional configuration entity. */ -public enum AnalyticsColumnType { +public enum AnalyticsDimensionType { STATIC, DYNAMIC; } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/model/AnalyticsTableColumn.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/model/AnalyticsTableColumn.java index 2f04beb6f675..537b473493e5 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/model/AnalyticsTableColumn.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/model/AnalyticsTableColumn.java @@ -74,7 +74,8 @@ public class AnalyticsTableColumn { @Builder.Default private final List indexColumns = List.of(); /** The column type indicates the column origin. */ - @Builder.Default private final AnalyticsColumnType columnType = AnalyticsColumnType.STATIC; + @Builder.Default + private final AnalyticsDimensionType dimensionType = AnalyticsDimensionType.STATIC; /** Date of creation of the underlying data dimension. */ private final Date created; @@ -104,8 +105,8 @@ public boolean isSkipIndex() { } /** Indicates whether the column type is set to a non-default value. */ - public boolean isStatic() { - return AnalyticsColumnType.STATIC == columnType; + public boolean isStaticDimension() { + return AnalyticsDimensionType.STATIC == dimensionType; } @Override diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsIndexHelper.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsIndexHelper.java index 07d05742ff2e..6d984a71c2b4 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsIndexHelper.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsIndexHelper.java @@ -154,7 +154,7 @@ private static void maybeAddTextLowerIndex( boolean isSingleColumn = indexColumns.size() == 1; if (column.getDataType() == TEXT - && !column.isStatic() + && !column.isStaticDimension() && isValidUid(columnName) && isSingleColumn) { String name = indexName + "_lower"; @@ -178,7 +178,7 @@ private static void maybeAddDateSortOrderIndex( boolean isSingleColumn = indexColumns.size() == 1; - if (column.getDataType() == TIMESTAMP && column.isStatic() && isSingleColumn) { + if (column.getDataType() == TIMESTAMP && column.isStaticDimension() && isSingleColumn) { indexes.add( Index.builder() .name(indexName + "_desc") From c1b20d73fd638ba6b029ef348c4ecebaa33acabd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 26 Nov 2024 16:35:26 +0100 Subject: [PATCH 019/170] fix: Update code --- .../main/java/org/hisp/dhis/analytics/common/ColumnHeader.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/analytics/common/ColumnHeader.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/analytics/common/ColumnHeader.java index 1e9e65e1681a..814be376f4db 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/analytics/common/ColumnHeader.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/analytics/common/ColumnHeader.java @@ -34,8 +34,9 @@ public enum ColumnHeader { GEOMETRY("geometry", "Geometry"), ENROLLMENT_DATE("enrollmentdate", "Enrollment date"), /** - * Deprecated since 2.42. Use @Link{#OCCURRED_DATE} instead. + * @deprecated use @Link{#OCCURRED_DATE} instead. */ + @Deprecated(since = "2.42") INCIDENT_DATE("incidentdate", "Incident date"), OCCURRED_DATE("occurreddate", "Occurred date"), STORED_BY("storedby", "Stored by"), From c51fb149780cadde2c72d81080fda8c8c83cd218 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 26 Nov 2024 17:03:00 +0100 Subject: [PATCH 020/170] fix: Update code --- .../dhis/common/DimensionalObjectUtils.java | 7 +- .../table/JdbcEventAnalyticsTableManager.java | 75 +++++++++++++------ 2 files changed, 55 insertions(+), 27 deletions(-) diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObjectUtils.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObjectUtils.java index f4a293029fea..a7786ad46813 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObjectUtils.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObjectUtils.java @@ -38,6 +38,9 @@ import static org.hisp.dhis.common.DimensionalObject.ITEM_SEP; import static org.hisp.dhis.common.DimensionalObject.OPTION_SEP; import static org.hisp.dhis.expression.ExpressionService.SYMBOL_WILDCARD; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -50,6 +53,7 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; +import lombok.NoArgsConstructor; import org.apache.commons.collections4.ListUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Triple; @@ -59,9 +63,6 @@ import org.hisp.dhis.eventvisualization.EventRepetition; import org.hisp.dhis.program.Program; import org.hisp.dhis.program.ProgramStage; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import lombok.NoArgsConstructor; /** * @author Lars Helge Overland diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 1016ca4f3dcf..766ae26af224 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -421,30 +421,8 @@ private List getColumns(Program program) { columns.addAll(getOrganisationUnitGroupSetColumns()); columns.addAll(getAttributeCategoryOptionGroupSetColumns()); columns.addAll(getPeriodTypeColumns("dps")); - - columns.addAll( - program.getAnalyticsDataElements().stream() - .map(de -> getColumnForDataElement(de, false)) - .flatMap(Collection::stream) - .toList()); - - columns.addAll( - program.getAnalyticsDataElementsWithLegendSet().stream() - .map(de -> getColumnForDataElement(de, true)) - .flatMap(Collection::stream) - .toList()); - - columns.addAll( - program.getNonConfidentialTrackedEntityAttributes().stream() - .map(tea -> getColumnForTrackedEntityAttribute(tea, false)) - .flatMap(Collection::stream) - .toList()); - - columns.addAll( - program.getNonConfidentialTrackedEntityAttributesWithLegendSet().stream() - .map(tea -> getColumnForTrackedEntityAttribute(tea, true)) - .flatMap(Collection::stream) - .toList()); + columns.addAll(getDataElementColumns(program)); + columns.addAll(getAttributeColumns(program)); if (program.isRegistration()) { columns.add(EventAnalyticsColumn.TRACKED_ENTITY); @@ -469,6 +447,27 @@ protected AnalyticsTableColumn getPartitionColumn() { .build(); } + /** + * Returns columns for data elements of the given program. + * + * @param program the {@link Program}. + * @return a list of {@link AnalyticsTableColumn}. + */ + private List getDataElementColumns(Program program) { + List columns = new ArrayList<>(); + columns.addAll( + program.getAnalyticsDataElements().stream() + .map(de -> getColumnForDataElement(de, false)) + .flatMap(Collection::stream) + .toList()); + columns.addAll( + program.getAnalyticsDataElementsWithLegendSet().stream() + .map(de -> getColumnForDataElement(de, true)) + .flatMap(Collection::stream) + .toList()); + return columns; + } + /** * Returns a column for the given data element. If the value type of the data element is {@link * ValueType#ORGANISATION_UNIT}, an extra column will be included. @@ -545,6 +544,34 @@ private List getColumnForOrgUnitDataElement( return columns; } + /** + * Returns columns for attributes of the given program. + * + * @param program the {@link Program}. + * @return a list of {@link AnalyticsTableColumn}. + */ + private List getAttributeColumns(Program program) { + List columns = new ArrayList<>(); + columns.addAll( + program.getNonConfidentialTrackedEntityAttributes().stream() + .map(tea -> getColumnForTrackedEntityAttribute(tea, false)) + .flatMap(Collection::stream) + .toList()); + columns.addAll( + program.getNonConfidentialTrackedEntityAttributesWithLegendSet().stream() + .map(tea -> getColumnForTrackedEntityAttribute(tea, true)) + .flatMap(Collection::stream) + .toList()); + return columns; + } + + /** + * Returns a list of columns based on the given attribute. + * + * @param attribute the {@link TrackedEntityAttribute}. + * @param withLegendSet indicates whether the attribute has a legend set. + * @return a list of {@link AnaylyticsTableColumn}. + */ private List getColumnForTrackedEntityAttribute( TrackedEntityAttribute attribute, boolean withLegendSet) { List columns = new ArrayList<>(); From a84ff584f1ed8b4b09a8f5a4da44c0519ad7bec7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 26 Nov 2024 17:24:54 +0100 Subject: [PATCH 021/170] fix: Update code --- .../org/hisp/dhis/category/CategoryCombo.java | 6 +++ .../table/JdbcEventAnalyticsTableManager.java | 45 +++++++++++-------- 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/category/CategoryCombo.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/category/CategoryCombo.java index b8b804117d5a..11246bd5533d 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/category/CategoryCombo.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/category/CategoryCombo.java @@ -27,6 +27,7 @@ */ package org.hisp.dhis.category; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; @@ -195,6 +196,11 @@ public boolean hasOptionCombos() { return optionCombos != null && !optionCombos.isEmpty(); } + @JsonIgnore + public List getDataDimensionCategories() { + return categories.stream().filter(Category::isDataDimension).toList(); + } + // ------------------------------------------------------------------------- // Logic // ------------------------------------------------------------------------- diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 766ae26af224..5848eaf4ff9e 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -398,24 +398,7 @@ private String getPartitionClause(AnalyticsTablePartition partition) { */ private List getColumns(Program program) { List columns = new ArrayList<>(fixedColumns); - - if (program.hasNonDefaultCategoryCombo()) { - List categories = program.getCategoryCombo().getCategories(); - - for (Category category : categories) { - if (category.isDataDimension()) { - columns.add( - AnalyticsTableColumn.builder() - .name(category.getUid()) - .dimensionType(AnalyticsDimensionType.DYNAMIC) - .dataType(CHARACTER_11) - .selectExpression("acs." + quote(category.getUid())) - .created(category.getCreated()) - .build()); - } - } - } - + columns.addAll(getAttributeCategoryColumns(program)); columns.addAll(getOrganisationUnitLevelColumns()); columns.add(getOrganisationUnitNameHierarchyColumn()); columns.addAll(getOrganisationUnitGroupSetColumns()); @@ -447,6 +430,32 @@ protected AnalyticsTableColumn getPartitionColumn() { .build(); } + /** + * Returns columns for attribute categories of the given program. + * + * @param program the {@link Program}. + * @return a list of {@link AnalyticsTableColumn}. + */ + private List getAttributeCategoryColumns(Program program) { + if (program.hasNonDefaultCategoryCombo()) { + List categories = program.getCategoryCombo().getDataDimensionCategories(); + + return categories.stream() + .map( + category -> + AnalyticsTableColumn.builder() + .name(category.getUid()) + .dimensionType(AnalyticsDimensionType.DYNAMIC) + .dataType(CHARACTER_11) + .selectExpression("acs." + quote(category.getUid())) + .created(category.getCreated()) + .build()) + .toList(); + } + + return List.of(); + } + /** * Returns columns for data elements of the given program. * From e3752118b29277e0b380c8681ec7bae074de7d1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 26 Nov 2024 17:26:00 +0100 Subject: [PATCH 022/170] fix: Update code --- .../dhis/analytics/table/JdbcEventAnalyticsTableManager.java | 1 - 1 file changed, 1 deletion(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 5848eaf4ff9e..08999c31c8ff 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -439,7 +439,6 @@ protected AnalyticsTableColumn getPartitionColumn() { private List getAttributeCategoryColumns(Program program) { if (program.hasNonDefaultCategoryCombo()) { List categories = program.getCategoryCombo().getDataDimensionCategories(); - return categories.stream() .map( category -> From 43ff1026061fc42900d7b95e4302e1eb05b05cab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 26 Nov 2024 18:08:34 +0100 Subject: [PATCH 023/170] fix: Update code --- .../table/AbstractEventJdbcTableManager.java | 63 ------------------ .../JdbcEnrollmentAnalyticsTableManager.java | 64 +++++++++++++++++++ 2 files changed, 64 insertions(+), 63 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 225991dcba23..f6ac6eddb560 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -27,18 +27,13 @@ */ package org.hisp.dhis.analytics.table; -import static org.hisp.dhis.analytics.table.model.Skip.SKIP; import static org.hisp.dhis.analytics.util.AnalyticsUtils.getClosingParentheses; -import static org.hisp.dhis.analytics.util.AnalyticsUtils.getColumnType; -import static org.hisp.dhis.db.model.DataType.TEXT; import static org.hisp.dhis.system.util.MathUtils.NUMERIC_LENIENT_REGEXP; -import java.util.ArrayList; import java.util.List; import java.util.Map; import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.partition.PartitionManager; -import org.hisp.dhis.analytics.table.model.AnalyticsDimensionType; import org.hisp.dhis.analytics.table.model.AnalyticsTableColumn; import org.hisp.dhis.analytics.table.model.AnalyticsTablePartition; import org.hisp.dhis.analytics.table.model.Skip; @@ -48,11 +43,9 @@ import org.hisp.dhis.common.ValueType; import org.hisp.dhis.commons.util.TextUtils; import org.hisp.dhis.dataapproval.DataApprovalLevelService; -import org.hisp.dhis.db.model.DataType; import org.hisp.dhis.db.sql.SqlBuilder; import org.hisp.dhis.organisationunit.OrganisationUnitService; import org.hisp.dhis.period.PeriodDataProvider; -import org.hisp.dhis.program.Program; import org.hisp.dhis.resourcetable.ResourceTableService; import org.hisp.dhis.setting.SystemSettingsProvider; import org.hisp.dhis.system.database.DatabaseInfoProvider; @@ -226,62 +219,6 @@ protected void populateTableInternal(AnalyticsTablePartition partition, String f invokeTimeAndLog(sql, "Populating table: '{}'", tableName); } - protected List getTrackedEntityAttributeColumns(Program program) { - List columns = new ArrayList<>(); - - for (TrackedEntityAttribute attribute : program.getNonConfidentialTrackedEntityAttributes()) { - DataType dataType = getColumnType(attribute.getValueType(), isSpatialSupport()); - String dataClause = - attribute.isNumericType() - ? getNumericClause() - : attribute.isDateType() ? getDateClause() : ""; - String select = getSelectExpressionForAttribute(attribute.getValueType(), "value"); - Skip skipIndex = skipIndex(attribute.getValueType(), attribute.hasOptionSet()); - - String sql = - replaceQualify( - """ - (select ${select} from ${trackedentityattributevalue} \ - where trackedentityid=en.trackedentityid \ - and trackedentityattributeid=${attributeId}\ - ${dataClause})${closingParentheses} as ${attributeUid}""", - Map.of( - "select", - select, - "attributeId", - String.valueOf(attribute.getId()), - "dataClause", - dataClause, - "closingParentheses", - getClosingParentheses(select), - "attributeUid", - quote(attribute.getUid()))); - columns.add( - AnalyticsTableColumn.builder() - .name(attribute.getUid()) - .dimensionType(AnalyticsDimensionType.DYNAMIC) - .dataType(dataType) - .selectExpression(sql) - .skipIndex(skipIndex) - .build()); - - if (attribute.getValueType().isOrganisationUnit()) { - String fromTypeSql = "ou.name from organisationunit ou where ou.uid = (select value"; - String ouNameSql = selectForInsert(attribute, fromTypeSql, dataClause); - - columns.add( - AnalyticsTableColumn.builder() - .name((attribute.getUid() + OU_NAME_COL_SUFFIX)) - .dimensionType(AnalyticsDimensionType.DYNAMIC) - .dataType(TEXT) - .selectExpression(ouNameSql) - .skipIndex(SKIP) - .build()); - } - } - return columns; - } - /** * The select statement used by the table population. * diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java index a6f0841dc2c0..139137fef9cb 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java @@ -27,6 +27,10 @@ */ package org.hisp.dhis.analytics.table; +import static org.hisp.dhis.analytics.table.model.Skip.SKIP; +import static org.hisp.dhis.analytics.util.AnalyticsUtils.getClosingParentheses; +import static org.hisp.dhis.analytics.util.AnalyticsUtils.getColumnType; +import static org.hisp.dhis.db.model.DataType.TEXT; import static org.hisp.dhis.util.DateUtils.toLongDate; import java.util.ArrayList; @@ -37,14 +41,17 @@ import org.hisp.dhis.analytics.AnalyticsTableType; import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; import org.hisp.dhis.analytics.partition.PartitionManager; +import org.hisp.dhis.analytics.table.model.AnalyticsDimensionType; import org.hisp.dhis.analytics.table.model.AnalyticsTable; import org.hisp.dhis.analytics.table.model.AnalyticsTableColumn; import org.hisp.dhis.analytics.table.model.AnalyticsTablePartition; +import org.hisp.dhis.analytics.table.model.Skip; import org.hisp.dhis.analytics.table.setting.AnalyticsTableSettings; import org.hisp.dhis.category.CategoryService; import org.hisp.dhis.common.IdentifiableObjectManager; import org.hisp.dhis.commons.collection.UniqueArrayList; import org.hisp.dhis.dataapproval.DataApprovalLevelService; +import org.hisp.dhis.db.model.DataType; import org.hisp.dhis.db.model.Logged; import org.hisp.dhis.db.sql.SqlBuilder; import org.hisp.dhis.organisationunit.OrganisationUnitService; @@ -53,6 +60,7 @@ import org.hisp.dhis.resourcetable.ResourceTableService; import org.hisp.dhis.setting.SystemSettingsProvider; import org.hisp.dhis.system.database.DatabaseInfoProvider; +import org.hisp.dhis.trackedentity.TrackedEntityAttribute; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; @@ -189,4 +197,60 @@ private List getColumns(Program program) { return filterDimensionColumns(columns); } + + private List getTrackedEntityAttributeColumns(Program program) { + List columns = new ArrayList<>(); + + for (TrackedEntityAttribute attribute : program.getNonConfidentialTrackedEntityAttributes()) { + DataType dataType = getColumnType(attribute.getValueType(), isSpatialSupport()); + String dataClause = + attribute.isNumericType() + ? getNumericClause() + : attribute.isDateType() ? getDateClause() : ""; + String select = getSelectExpressionForAttribute(attribute.getValueType(), "value"); + Skip skipIndex = skipIndex(attribute.getValueType(), attribute.hasOptionSet()); + + String sql = + replaceQualify( + """ + (select ${select} from ${trackedentityattributevalue} \ + where trackedentityid=en.trackedentityid \ + and trackedentityattributeid=${attributeId}\ + ${dataClause})${closingParentheses} as ${attributeUid}""", + Map.of( + "select", + select, + "attributeId", + String.valueOf(attribute.getId()), + "dataClause", + dataClause, + "closingParentheses", + getClosingParentheses(select), + "attributeUid", + quote(attribute.getUid()))); + columns.add( + AnalyticsTableColumn.builder() + .name(attribute.getUid()) + .dimensionType(AnalyticsDimensionType.DYNAMIC) + .dataType(dataType) + .selectExpression(sql) + .skipIndex(skipIndex) + .build()); + + if (attribute.getValueType().isOrganisationUnit()) { + String fromTypeSql = "ou.name from organisationunit ou where ou.uid = (select value"; + String ouNameSql = selectForInsert(attribute, fromTypeSql, dataClause); + + columns.add( + AnalyticsTableColumn.builder() + .name((attribute.getUid() + OU_NAME_COL_SUFFIX)) + .dimensionType(AnalyticsDimensionType.DYNAMIC) + .dataType(TEXT) + .selectExpression(ouNameSql) + .skipIndex(SKIP) + .build()); + } + } + return columns; + } } From 2cab8fce5e176bae188f52e6052b2d8b276baa0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 26 Nov 2024 18:12:17 +0100 Subject: [PATCH 024/170] fix: Update code --- .../JdbcEnrollmentAnalyticsTableManager.java | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java index 139137fef9cb..59be5e960ff0 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java @@ -187,13 +187,7 @@ private List getColumns(Program program) { columns.addAll(getOrganisationUnitGroupSetColumns()); columns.addAll(getPeriodTypeColumns("dps")); columns.addAll(getTrackedEntityAttributeColumns(program)); - - if (program.isRegistration()) { - columns.add(EnrollmentAnalyticsColumn.TRACKED_ENTITY); - if (sqlBuilder.supportsGeospatialData()) { - columns.add(EnrollmentAnalyticsColumn.TRACKED_ENTITY_GEOMETRY); - } - } + columns.addAll(getTrackedEntityColumns(program)); return filterDimensionColumns(columns); } @@ -253,4 +247,23 @@ private List getTrackedEntityAttributeColumns(Program prog } return columns; } + + /** + * Returns a list of tracked entity {@link AnalyticsTableColumn}. + * + * @param program the {@link Program}. + * @return a list of {@link AnalyticsTableColumn}. + */ + private List getTrackedEntityColumns(Program program) { + List columns = new ArrayList<>(); + + if (program.isRegistration()) { + columns.add(EnrollmentAnalyticsColumn.TRACKED_ENTITY); + if (sqlBuilder.supportsGeospatialData()) { + columns.add(EnrollmentAnalyticsColumn.TRACKED_ENTITY_GEOMETRY); + } + } + + return columns; + } } From adc1599bf282ed585ee2606a4eeb1ae2086f1734 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 26 Nov 2024 18:13:05 +0100 Subject: [PATCH 025/170] fix: Update code --- .../table/JdbcEnrollmentAnalyticsTableManager.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java index 59be5e960ff0..7a451dc31086 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java @@ -192,6 +192,12 @@ private List getColumns(Program program) { return filterDimensionColumns(columns); } + /** + * Returns a list of tracked entity attribute {@link AnalyticsTableColumn}. + * + * @param program the {@link Program}. + * @return a list of {@link AnalyticsTableColumn}. + */ private List getTrackedEntityAttributeColumns(Program program) { List columns = new ArrayList<>(); From 793b9fa1fc0079166ad149d4369f4ecaa6fbd7af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 26 Nov 2024 20:12:54 +0100 Subject: [PATCH 026/170] fix: Update code --- .../JdbcTrackedEntityAnalyticsTableManager.java | 8 +++----- ...kedEntityEnrollmentsAnalyticsTableManager.java | 15 +++++++++++++++ ...cTrackedEntityEventsAnalyticsTableManager.java | 11 ++++++++--- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java index bec6c798dbba..b7781c3ea125 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java @@ -237,15 +237,15 @@ private List getColumns( private Stream getAllTrackedEntityAttributes( TrackedEntityType trackedEntityType, Map> programsByTetUid) { - // Given TET has program(s) defined. + // Given TET has program(s) defined if (programsByTetUid.containsKey(trackedEntityType.getUid())) { - // Programs defined for TET -> get attr from program and TET. + // Programs defined for TET -> get attr from program and TET return getAllTrackedEntityAttributesByPrograms( trackedEntityType, programsByTetUid.get(trackedEntityType.getUid())); } - // No programs defined for TET -> get only attributes from TET. + // No programs defined for TET -> get only attributes from TET return getAllTrackedEntityAttributesByEntityType(trackedEntityType); } @@ -297,9 +297,7 @@ private String castBasedOnType(ValueType valueType, String columnName) { private Stream getAllTrackedEntityAttributesByPrograms( TrackedEntityType trackedEntityType, List programs) { return Stream.concat( - /* all attributes of programs */ trackedEntityAttributeService.getProgramTrackedEntityAttributes(programs).stream(), - /* all attributes of the trackedEntityType */ getAllTrackedEntityAttributesByEntityType(trackedEntityType)) .distinct(); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEnrollmentsAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEnrollmentsAnalyticsTableManager.java index 95a57aaf8168..fc9f6f1b510a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEnrollmentsAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEnrollmentsAnalyticsTableManager.java @@ -287,6 +287,11 @@ and ev.status in (${statuses})) \ invokeTimeAndLog(sql.toString(), "Populating table: '{}'", tableName); } + /** + * Returns a list of fixed columns. + * + * @return a list of {@link AnalyticsTableColumn}. + */ private List getFixedCols() { List columns = new ArrayList<>(); columns.addAll(FIXED_COLS); @@ -296,6 +301,11 @@ private List getFixedCols() { return columns; } + /** + * Returns a list of geospatial columns. + * + * @return a list of {@link AnalyticsTableColumn}. + */ private List getGeospatialCols() { return List.of( @@ -319,6 +329,11 @@ private List getGeospatialCols() { .build()); } + /** + * Returns a partition column. + * + * @return an {@link AnalyticsTableColumn}. + */ private AnalyticsTableColumn getPartitionColumn() { return AnalyticsTableColumn.builder() .name("year") diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManager.java index 904374494c13..f3055b6db7bc 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManager.java @@ -193,9 +193,8 @@ public class JdbcTrackedEntityEventsAnalyticsTableManager extends AbstractJdbcTa .selectExpression("ous.level") .build()); - private static final String AND = " and ("; - private final TrackedEntityTypeService trackedEntityTypeService; + private final AnalyticsSqlBuilder analyticsSqlBuilder; public JdbcTrackedEntityEventsAnalyticsTableManager( @@ -340,7 +339,8 @@ private List getDataYears(AnalyticsTableUpdateParams params, TrackedEnt "tetId", String.valueOf(tet.getId())))); if (params.getFromDate() != null) { - sql.append(AND + eventDateExpression + ") >= '" + toMediumDate(params.getFromDate()) + "'"); + sql.append( + " and (" + eventDateExpression + ") >= '" + toMediumDate(params.getFromDate()) + "'"); } List availableDataYears = @@ -451,6 +451,11 @@ private String getPartitionClause(AnalyticsTablePartition partition) { : emptyIfTrue(partitionFilter, sqlBuilder.supportsDeclarativePartitioning()); } + /** + * Returns a partition column. + * + * @return an {@link AnalyticsTableColumn}. + */ private AnalyticsTableColumn getPartitionColumn() { return AnalyticsTableColumn.builder() .name("year") From 105b514d465124df0edfe6fb7da6a8e3b203dec9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 26 Nov 2024 20:15:51 +0100 Subject: [PATCH 027/170] fix: Update code --- .../dhis/analytics/table/JdbcEventAnalyticsTableManager.java | 1 - 1 file changed, 1 deletion(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 9182cfc490d5..72e93ba076ee 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -414,7 +414,6 @@ private List getColumns(Program program) { } } if (sqlBuilder.supportsDeclarativePartitioning()) { - // Add the year column required for declarative partitioning columns.add(getPartitionColumn()); } From 45d734e4f9b38b548420d3d67e35779b3caf84ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 26 Nov 2024 20:32:08 +0100 Subject: [PATCH 028/170] fix: Update code --- .../table/JdbcEventAnalyticsTableManager.java | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 72e93ba076ee..3d7844f9091e 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -50,6 +50,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.AnalyticsTableType; @@ -332,6 +333,7 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti Integer latestDataYear = availableDataYears.get(availableDataYears.size() - 1); Program program = partition.getMasterTable().getProgram(); String partitionClause = getPartitionClause(partition); + String attributeJoinClause = getAttributeJoinClause(program); String fromClause = replaceQualify( @@ -350,6 +352,7 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti left join ${organisationunit} enrollmentou on en.organisationunitid=enrollmentou.organisationunitid \ inner join analytics_rs_categorystructure acs on ev.attributeoptioncomboid=acs.categoryoptioncomboid \ left join analytics_rs_dateperiodstructure dps on cast(${eventDateExpression} as date)=dps.dateperiod \ + ${attributeJoinClause}\ where ev.lastupdated < '${startTime}' ${partitionClause} \ and pr.programid=${programId} \ and ev.organisationunitid is not null \ @@ -362,6 +365,7 @@ and ev.status in (${exportableEventStatues}) \ "eventDateMonth", sqlBuilder.dateTrunc("month", eventDateExpression), "eventDateExpression", eventDateExpression, "partitionClause", partitionClause, + "attributeJoinClause", attributeJoinClause, "startTime", toLongDate(params.getStartTime()), "programId", String.valueOf(program.getId()), "firstDataYear", String.valueOf(firstDataYear), @@ -371,6 +375,19 @@ and ev.status in (${exportableEventStatues}) \ populateTableInternal(partition, fromClause); } + private String getAttributeJoinClause(Program program) { + String template = + """ + left join ${trackedentityattributevalue} ${uid} \ + on en.trackedentityid=${uid}.trackedentityid \ + and ${uid}.trackedentityattributeid = ${uid}"""; + + return getAttributeColumns(program).stream() + .map(col -> replaceQualify(template, Map.of("uid", quote(col.getName())))) + .collect(Collectors.joining(" ")) + + " "; + } + /** * Returns a partition SQL clause. * @@ -420,7 +437,7 @@ private List getColumns(Program program) { return filterDimensionColumns(columns); } - protected AnalyticsTableColumn getPartitionColumn() { + private AnalyticsTableColumn getPartitionColumn() { return AnalyticsTableColumn.builder() .name("year") .dataType(INTEGER) @@ -815,6 +832,7 @@ private List getDataYears( "fromDate", toMediumDate(params.getFromDate()))) : EMPTY; + String sql = replaceQualify( """ From 793a55bc4e608ee49af2aba5fbbd0631cc4ec4d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 26 Nov 2024 20:33:14 +0100 Subject: [PATCH 029/170] fix: Update code --- .../analytics/table/JdbcEventAnalyticsTableManager.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 3d7844f9091e..312917e0ff06 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -375,6 +375,12 @@ and ev.status in (${exportableEventStatues}) \ populateTableInternal(partition, fromClause); } + /** + * Returns a join clause for attribute value for every attribute of the given program. + * + * @param program the {@link Program}. + * @return a join clause. + */ private String getAttributeJoinClause(Program program) { String template = """ From 0640b81d059c8f8c0dddbe03569c12d20278428f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 26 Nov 2024 20:34:39 +0100 Subject: [PATCH 030/170] fix: Update code --- .../table/JdbcEventAnalyticsTableManager.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 312917e0ff06..ca676cb4450a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -41,7 +41,6 @@ import static org.hisp.dhis.db.model.DataType.TEXT; import static org.hisp.dhis.util.DateUtils.toLongDate; import static org.hisp.dhis.util.DateUtils.toMediumDate; - import java.time.Year; import java.util.ArrayList; import java.util.Collection; @@ -51,7 +50,6 @@ import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.AnalyticsTableType; import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; @@ -90,6 +88,7 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.util.Assert; +import lombok.extern.slf4j.Slf4j; /** * @author Lars Helge Overland @@ -388,10 +387,11 @@ private String getAttributeJoinClause(Program program) { on en.trackedentityid=${uid}.trackedentityid \ and ${uid}.trackedentityattributeid = ${uid}"""; - return getAttributeColumns(program).stream() - .map(col -> replaceQualify(template, Map.of("uid", quote(col.getName())))) - .collect(Collectors.joining(" ")) - + " "; + String sql = getAttributeColumns(program).stream() + .map(col -> replaceQualify(template, Map.of("uid", quote(col.getName())))) + .collect(Collectors.joining(" ")); + + return sql + " "; } /** From f33bd881014f02dbba408653ac01fdc64e19a7a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 26 Nov 2024 20:34:52 +0100 Subject: [PATCH 031/170] fix: Update code --- .../table/JdbcEventAnalyticsTableManager.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index ca676cb4450a..bb3ae7df2ae7 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -41,6 +41,7 @@ import static org.hisp.dhis.db.model.DataType.TEXT; import static org.hisp.dhis.util.DateUtils.toLongDate; import static org.hisp.dhis.util.DateUtils.toMediumDate; + import java.time.Year; import java.util.ArrayList; import java.util.Collection; @@ -50,6 +51,7 @@ import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.AnalyticsTableType; import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; @@ -88,7 +90,6 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.util.Assert; -import lombok.extern.slf4j.Slf4j; /** * @author Lars Helge Overland @@ -332,7 +333,7 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti Integer latestDataYear = availableDataYears.get(availableDataYears.size() - 1); Program program = partition.getMasterTable().getProgram(); String partitionClause = getPartitionClause(partition); - String attributeJoinClause = getAttributeJoinClause(program); + String attributeJoinClause = getAttributeValueJoinClause(program); String fromClause = replaceQualify( @@ -380,17 +381,18 @@ and ev.status in (${exportableEventStatues}) \ * @param program the {@link Program}. * @return a join clause. */ - private String getAttributeJoinClause(Program program) { + private String getAttributeValueJoinClause(Program program) { String template = """ left join ${trackedentityattributevalue} ${uid} \ on en.trackedentityid=${uid}.trackedentityid \ and ${uid}.trackedentityattributeid = ${uid}"""; - String sql = getAttributeColumns(program).stream() - .map(col -> replaceQualify(template, Map.of("uid", quote(col.getName())))) - .collect(Collectors.joining(" ")); - + String sql = + getAttributeColumns(program).stream() + .map(col -> replaceQualify(template, Map.of("uid", quote(col.getName())))) + .collect(Collectors.joining(" ")); + return sql + " "; } From 3a6b574f1cd61218e2412bf2ed9bae60f11cbe29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 26 Nov 2024 20:35:09 +0100 Subject: [PATCH 032/170] fix: Update code --- .../table/JdbcEventAnalyticsTableManager.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index bb3ae7df2ae7..9a5e40d58302 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -41,7 +41,6 @@ import static org.hisp.dhis.db.model.DataType.TEXT; import static org.hisp.dhis.util.DateUtils.toLongDate; import static org.hisp.dhis.util.DateUtils.toMediumDate; - import java.time.Year; import java.util.ArrayList; import java.util.Collection; @@ -51,7 +50,6 @@ import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.AnalyticsTableType; import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; @@ -90,6 +88,7 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.util.Assert; +import lombok.extern.slf4j.Slf4j; /** * @author Lars Helge Overland @@ -388,10 +387,9 @@ private String getAttributeValueJoinClause(Program program) { on en.trackedentityid=${uid}.trackedentityid \ and ${uid}.trackedentityattributeid = ${uid}"""; - String sql = - getAttributeColumns(program).stream() - .map(col -> replaceQualify(template, Map.of("uid", quote(col.getName())))) - .collect(Collectors.joining(" ")); + String sql =getAttributeColumns(program).stream() + .map(col -> replaceQualify(template, Map.of("uid", quote(col.getName())))) + .collect(Collectors.joining(" ")); return sql + " "; } From 7ac31de85f9d8b3304fff778c714c86ae815d46f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 26 Nov 2024 21:46:45 +0100 Subject: [PATCH 033/170] fix: Update code --- .../table/JdbcEventAnalyticsTableManager.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 9a5e40d58302..5975f5da5642 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -385,13 +385,12 @@ private String getAttributeValueJoinClause(Program program) { """ left join ${trackedentityattributevalue} ${uid} \ on en.trackedentityid=${uid}.trackedentityid \ - and ${uid}.trackedentityattributeid = ${uid}"""; + and ${uid}.trackedentityattributeid = ${id}\s"""; - String sql =getAttributeColumns(program).stream() - .map(col -> replaceQualify(template, Map.of("uid", quote(col.getName())))) - .collect(Collectors.joining(" ")); - - return sql + " "; + return program.getNonConfidentialTrackedEntityAttributes().stream() + .map(at -> replaceQualify(template, Map.of( + "uid", quote(at.getUid()), "id", String.valueOf(at.getId())))) + .collect(Collectors.joining()); } /** From cd9c2ed9e295935de711db36de415055a54d7847 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 26 Nov 2024 21:47:14 +0100 Subject: [PATCH 034/170] fix: Update code --- .../analytics/table/JdbcEventAnalyticsTableManager.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 5975f5da5642..ed53f54e9aed 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -41,6 +41,7 @@ import static org.hisp.dhis.db.model.DataType.TEXT; import static org.hisp.dhis.util.DateUtils.toLongDate; import static org.hisp.dhis.util.DateUtils.toMediumDate; + import java.time.Year; import java.util.ArrayList; import java.util.Collection; @@ -50,6 +51,7 @@ import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.AnalyticsTableType; import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; @@ -88,7 +90,6 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.util.Assert; -import lombok.extern.slf4j.Slf4j; /** * @author Lars Helge Overland @@ -388,8 +389,10 @@ private String getAttributeValueJoinClause(Program program) { and ${uid}.trackedentityattributeid = ${id}\s"""; return program.getNonConfidentialTrackedEntityAttributes().stream() - .map(at -> replaceQualify(template, Map.of( - "uid", quote(at.getUid()), "id", String.valueOf(at.getId())))) + .map( + at -> + replaceQualify( + template, Map.of("uid", quote(at.getUid()), "id", String.valueOf(at.getId())))) .collect(Collectors.joining()); } From 9cf4321ad9f7efc4d4d771261aef7172b14174f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 26 Nov 2024 22:42:47 +0100 Subject: [PATCH 035/170] fix: Update code --- .../table/JdbcEventAnalyticsTableManager.java | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index b5ca5601adc3..0ab1b5a3b676 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -66,6 +66,7 @@ import org.hisp.dhis.calendar.Calendar; import org.hisp.dhis.category.Category; import org.hisp.dhis.category.CategoryService; +import org.hisp.dhis.common.IdentifiableObject; import org.hisp.dhis.common.IdentifiableObjectManager; import org.hisp.dhis.common.ValueType; import org.hisp.dhis.commons.collection.ListUtils; @@ -389,13 +390,22 @@ private String getAttributeValueJoinClause(Program program) { and ${uid}.trackedentityattributeid = ${id}\s"""; return program.getNonConfidentialTrackedEntityAttributes().stream() - .map( - at -> - replaceQualify( - template, Map.of("uid", quote(at.getUid()), "id", String.valueOf(at.getId())))) + .map(at -> replaceQualify(template, toVariableMap(at))) .collect(Collectors.joining()); } + /** + * Returns a variable map. + * + * @param object the object. + * @return a {@link Map}. + */ + private Map toVariableMap(IdentifiableObject object) { + return Map.of( + "id", String.valueOf(object.getId()), + "uid", quote(object.getUid())); + } + /** * Returns a partition SQL clause. * From df69357740aee75be7d76bb168b07b8913ba3b30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 26 Nov 2024 22:43:27 +0100 Subject: [PATCH 036/170] fix: Update code --- .../dhis/analytics/table/JdbcEventAnalyticsTableManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 0ab1b5a3b676..23270d34231f 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -390,7 +390,7 @@ private String getAttributeValueJoinClause(Program program) { and ${uid}.trackedentityattributeid = ${id}\s"""; return program.getNonConfidentialTrackedEntityAttributes().stream() - .map(at -> replaceQualify(template, toVariableMap(at))) + .map(attribute -> replaceQualify(template, toVariableMap(attribute))) .collect(Collectors.joining()); } From 007fabb4a806cf5f5703d5f5a9a87f6c35662801 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Fri, 29 Nov 2024 09:47:03 +0100 Subject: [PATCH 037/170] fix: Update code --- .../dhis/analytics/event/data/TimeFieldSqlRenderer.java | 6 +++--- .../hisp/dhis/program/function/ProgramCountFunction.java | 2 +- .../hisp/dhis/program/function/ProgramMinMaxFunction.java | 2 +- .../dhis/parser/expression/CommonExpressionVisitor.java | 6 +++--- .../expression/statement/DefaultSqlStatementBuilder.java} | 4 ++-- .../dhis/parser/expression/statement}/StatementBuilder.java | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) rename dhis-2/dhis-support/{dhis-support-jdbc/src/main/java/org/hisp/dhis/jdbc/PostgreSqlStatementBuilder.java => dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/statement/DefaultSqlStatementBuilder.java} (99%) rename dhis-2/dhis-support/{dhis-support-jdbc/src/main/java/org/hisp/dhis/jdbc => dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/statement}/StatementBuilder.java (99%) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/TimeFieldSqlRenderer.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/TimeFieldSqlRenderer.java index 8cccb77fdc90..39029174670b 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/TimeFieldSqlRenderer.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/TimeFieldSqlRenderer.java @@ -51,15 +51,15 @@ import org.hisp.dhis.common.DateRange; import org.hisp.dhis.common.DimensionalItemObject; import org.hisp.dhis.db.sql.SqlBuilder; -import org.hisp.dhis.jdbc.PostgreSqlStatementBuilder; -import org.hisp.dhis.jdbc.StatementBuilder; +import org.hisp.dhis.parser.expression.statement.DefaultSqlStatementBuilder; +import org.hisp.dhis.parser.expression.statement.StatementBuilder; import org.hisp.dhis.period.Period; import org.hisp.dhis.period.PeriodType; /** Provides methods targeting the generation of SQL statements for periods and time fields. */ public abstract class TimeFieldSqlRenderer { protected final SqlBuilder sqlBuilder; - protected final StatementBuilder statementBuilder = new PostgreSqlStatementBuilder(); + protected final StatementBuilder statementBuilder = new DefaultSqlStatementBuilder(); protected TimeFieldSqlRenderer(SqlBuilder sqlBuilder) { this.sqlBuilder = sqlBuilder; diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/function/ProgramCountFunction.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/function/ProgramCountFunction.java index 62a4978b05b7..be852b9cfb3b 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/function/ProgramCountFunction.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/function/ProgramCountFunction.java @@ -30,10 +30,10 @@ import java.util.Date; import org.hisp.dhis.analytics.AnalyticsConstants; import org.hisp.dhis.antlr.ParserExceptionWithoutContext; -import org.hisp.dhis.jdbc.StatementBuilder; import org.hisp.dhis.parser.expression.CommonExpressionVisitor; import org.hisp.dhis.parser.expression.ProgramExpressionParams; import org.hisp.dhis.parser.expression.antlr.ExpressionParser.ExprContext; +import org.hisp.dhis.parser.expression.statement.StatementBuilder; import org.hisp.dhis.program.ProgramExpressionItem; import org.hisp.dhis.program.ProgramIndicator; import org.hisp.dhis.program.dataitem.ProgramItemStageElement; diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/function/ProgramMinMaxFunction.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/function/ProgramMinMaxFunction.java index f37ae35271f0..efa538adce0e 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/function/ProgramMinMaxFunction.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/function/ProgramMinMaxFunction.java @@ -29,10 +29,10 @@ import java.util.Date; import org.hisp.dhis.analytics.AnalyticsConstants; -import org.hisp.dhis.jdbc.StatementBuilder; import org.hisp.dhis.parser.expression.CommonExpressionVisitor; import org.hisp.dhis.parser.expression.ProgramExpressionParams; import org.hisp.dhis.parser.expression.antlr.ExpressionParser.ExprContext; +import org.hisp.dhis.parser.expression.statement.StatementBuilder; import org.hisp.dhis.program.AnalyticsType; import org.hisp.dhis.program.ProgramExpressionItem; import org.hisp.dhis.program.ProgramIndicator; diff --git a/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/CommonExpressionVisitor.java b/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/CommonExpressionVisitor.java index 78f832cc96de..684adbb67a8d 100644 --- a/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/CommonExpressionVisitor.java +++ b/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/CommonExpressionVisitor.java @@ -46,9 +46,9 @@ import org.hisp.dhis.expression.ExpressionInfo; import org.hisp.dhis.expression.ExpressionParams; import org.hisp.dhis.i18n.I18n; -import org.hisp.dhis.jdbc.PostgreSqlStatementBuilder; -import org.hisp.dhis.jdbc.StatementBuilder; import org.hisp.dhis.parser.expression.antlr.ExpressionParser.ExprContext; +import org.hisp.dhis.parser.expression.statement.DefaultSqlStatementBuilder; +import org.hisp.dhis.parser.expression.statement.StatementBuilder; import org.hisp.dhis.program.ProgramIndicatorService; import org.hisp.dhis.program.ProgramStageService; import org.hisp.dhis.trackedentity.TrackedEntityAttributeService; @@ -62,7 +62,7 @@ @Setter @Builder(toBuilder = true) public class CommonExpressionVisitor extends AntlrExpressionVisitor { - private final StatementBuilder statementBuilder = new PostgreSqlStatementBuilder(); + private final StatementBuilder statementBuilder = new DefaultSqlStatementBuilder(); private IdentifiableObjectManager idObjectManager; diff --git a/dhis-2/dhis-support/dhis-support-jdbc/src/main/java/org/hisp/dhis/jdbc/PostgreSqlStatementBuilder.java b/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/statement/DefaultSqlStatementBuilder.java similarity index 99% rename from dhis-2/dhis-support/dhis-support-jdbc/src/main/java/org/hisp/dhis/jdbc/PostgreSqlStatementBuilder.java rename to dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/statement/DefaultSqlStatementBuilder.java index a5248d65eed1..f6a6a0e1eef0 100644 --- a/dhis-2/dhis-support/dhis-support-jdbc/src/main/java/org/hisp/dhis/jdbc/PostgreSqlStatementBuilder.java +++ b/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/statement/DefaultSqlStatementBuilder.java @@ -25,7 +25,7 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.hisp.dhis.jdbc; +package org.hisp.dhis.parser.expression.statement; import static java.lang.String.format; import static org.apache.commons.lang3.StringUtils.SPACE; @@ -50,7 +50,7 @@ * @author Lars Helge Overland */ @NoArgsConstructor -public class PostgreSqlStatementBuilder implements StatementBuilder { +public class DefaultSqlStatementBuilder implements StatementBuilder { protected static final String QUOTE = "\""; protected static final String SINGLE_QUOTE = "'"; diff --git a/dhis-2/dhis-support/dhis-support-jdbc/src/main/java/org/hisp/dhis/jdbc/StatementBuilder.java b/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/statement/StatementBuilder.java similarity index 99% rename from dhis-2/dhis-support/dhis-support-jdbc/src/main/java/org/hisp/dhis/jdbc/StatementBuilder.java rename to dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/statement/StatementBuilder.java index eb239c2986a3..e51f1ac6b58c 100644 --- a/dhis-2/dhis-support/dhis-support-jdbc/src/main/java/org/hisp/dhis/jdbc/StatementBuilder.java +++ b/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/statement/StatementBuilder.java @@ -25,7 +25,7 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.hisp.dhis.jdbc; +package org.hisp.dhis.parser.expression.statement; import java.util.Date; import org.hisp.dhis.program.AnalyticsPeriodBoundary; From 8af5d48e34c25984d8ff7df2b0d248434ad18be0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Fri, 29 Nov 2024 09:48:37 +0100 Subject: [PATCH 038/170] Revert "fix: Update code" This reverts commit 007fabb4a806cf5f5703d5f5a9a87f6c35662801. --- .../dhis/analytics/event/data/TimeFieldSqlRenderer.java | 6 +++--- .../hisp/dhis/program/function/ProgramCountFunction.java | 2 +- .../hisp/dhis/program/function/ProgramMinMaxFunction.java | 2 +- .../dhis/parser/expression/CommonExpressionVisitor.java | 6 +++--- .../org/hisp/dhis/jdbc/PostgreSqlStatementBuilder.java} | 4 ++-- .../src/main/java/org/hisp/dhis/jdbc}/StatementBuilder.java | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) rename dhis-2/dhis-support/{dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/statement/DefaultSqlStatementBuilder.java => dhis-support-jdbc/src/main/java/org/hisp/dhis/jdbc/PostgreSqlStatementBuilder.java} (99%) rename dhis-2/dhis-support/{dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/statement => dhis-support-jdbc/src/main/java/org/hisp/dhis/jdbc}/StatementBuilder.java (99%) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/TimeFieldSqlRenderer.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/TimeFieldSqlRenderer.java index 39029174670b..8cccb77fdc90 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/TimeFieldSqlRenderer.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/TimeFieldSqlRenderer.java @@ -51,15 +51,15 @@ import org.hisp.dhis.common.DateRange; import org.hisp.dhis.common.DimensionalItemObject; import org.hisp.dhis.db.sql.SqlBuilder; -import org.hisp.dhis.parser.expression.statement.DefaultSqlStatementBuilder; -import org.hisp.dhis.parser.expression.statement.StatementBuilder; +import org.hisp.dhis.jdbc.PostgreSqlStatementBuilder; +import org.hisp.dhis.jdbc.StatementBuilder; import org.hisp.dhis.period.Period; import org.hisp.dhis.period.PeriodType; /** Provides methods targeting the generation of SQL statements for periods and time fields. */ public abstract class TimeFieldSqlRenderer { protected final SqlBuilder sqlBuilder; - protected final StatementBuilder statementBuilder = new DefaultSqlStatementBuilder(); + protected final StatementBuilder statementBuilder = new PostgreSqlStatementBuilder(); protected TimeFieldSqlRenderer(SqlBuilder sqlBuilder) { this.sqlBuilder = sqlBuilder; diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/function/ProgramCountFunction.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/function/ProgramCountFunction.java index be852b9cfb3b..62a4978b05b7 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/function/ProgramCountFunction.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/function/ProgramCountFunction.java @@ -30,10 +30,10 @@ import java.util.Date; import org.hisp.dhis.analytics.AnalyticsConstants; import org.hisp.dhis.antlr.ParserExceptionWithoutContext; +import org.hisp.dhis.jdbc.StatementBuilder; import org.hisp.dhis.parser.expression.CommonExpressionVisitor; import org.hisp.dhis.parser.expression.ProgramExpressionParams; import org.hisp.dhis.parser.expression.antlr.ExpressionParser.ExprContext; -import org.hisp.dhis.parser.expression.statement.StatementBuilder; import org.hisp.dhis.program.ProgramExpressionItem; import org.hisp.dhis.program.ProgramIndicator; import org.hisp.dhis.program.dataitem.ProgramItemStageElement; diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/function/ProgramMinMaxFunction.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/function/ProgramMinMaxFunction.java index efa538adce0e..f37ae35271f0 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/function/ProgramMinMaxFunction.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/function/ProgramMinMaxFunction.java @@ -29,10 +29,10 @@ import java.util.Date; import org.hisp.dhis.analytics.AnalyticsConstants; +import org.hisp.dhis.jdbc.StatementBuilder; import org.hisp.dhis.parser.expression.CommonExpressionVisitor; import org.hisp.dhis.parser.expression.ProgramExpressionParams; import org.hisp.dhis.parser.expression.antlr.ExpressionParser.ExprContext; -import org.hisp.dhis.parser.expression.statement.StatementBuilder; import org.hisp.dhis.program.AnalyticsType; import org.hisp.dhis.program.ProgramExpressionItem; import org.hisp.dhis.program.ProgramIndicator; diff --git a/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/CommonExpressionVisitor.java b/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/CommonExpressionVisitor.java index 684adbb67a8d..78f832cc96de 100644 --- a/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/CommonExpressionVisitor.java +++ b/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/CommonExpressionVisitor.java @@ -46,9 +46,9 @@ import org.hisp.dhis.expression.ExpressionInfo; import org.hisp.dhis.expression.ExpressionParams; import org.hisp.dhis.i18n.I18n; +import org.hisp.dhis.jdbc.PostgreSqlStatementBuilder; +import org.hisp.dhis.jdbc.StatementBuilder; import org.hisp.dhis.parser.expression.antlr.ExpressionParser.ExprContext; -import org.hisp.dhis.parser.expression.statement.DefaultSqlStatementBuilder; -import org.hisp.dhis.parser.expression.statement.StatementBuilder; import org.hisp.dhis.program.ProgramIndicatorService; import org.hisp.dhis.program.ProgramStageService; import org.hisp.dhis.trackedentity.TrackedEntityAttributeService; @@ -62,7 +62,7 @@ @Setter @Builder(toBuilder = true) public class CommonExpressionVisitor extends AntlrExpressionVisitor { - private final StatementBuilder statementBuilder = new DefaultSqlStatementBuilder(); + private final StatementBuilder statementBuilder = new PostgreSqlStatementBuilder(); private IdentifiableObjectManager idObjectManager; diff --git a/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/statement/DefaultSqlStatementBuilder.java b/dhis-2/dhis-support/dhis-support-jdbc/src/main/java/org/hisp/dhis/jdbc/PostgreSqlStatementBuilder.java similarity index 99% rename from dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/statement/DefaultSqlStatementBuilder.java rename to dhis-2/dhis-support/dhis-support-jdbc/src/main/java/org/hisp/dhis/jdbc/PostgreSqlStatementBuilder.java index f6a6a0e1eef0..a5248d65eed1 100644 --- a/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/statement/DefaultSqlStatementBuilder.java +++ b/dhis-2/dhis-support/dhis-support-jdbc/src/main/java/org/hisp/dhis/jdbc/PostgreSqlStatementBuilder.java @@ -25,7 +25,7 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.hisp.dhis.parser.expression.statement; +package org.hisp.dhis.jdbc; import static java.lang.String.format; import static org.apache.commons.lang3.StringUtils.SPACE; @@ -50,7 +50,7 @@ * @author Lars Helge Overland */ @NoArgsConstructor -public class DefaultSqlStatementBuilder implements StatementBuilder { +public class PostgreSqlStatementBuilder implements StatementBuilder { protected static final String QUOTE = "\""; protected static final String SINGLE_QUOTE = "'"; diff --git a/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/statement/StatementBuilder.java b/dhis-2/dhis-support/dhis-support-jdbc/src/main/java/org/hisp/dhis/jdbc/StatementBuilder.java similarity index 99% rename from dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/statement/StatementBuilder.java rename to dhis-2/dhis-support/dhis-support-jdbc/src/main/java/org/hisp/dhis/jdbc/StatementBuilder.java index e51f1ac6b58c..eb239c2986a3 100644 --- a/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/statement/StatementBuilder.java +++ b/dhis-2/dhis-support/dhis-support-jdbc/src/main/java/org/hisp/dhis/jdbc/StatementBuilder.java @@ -25,7 +25,7 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.hisp.dhis.parser.expression.statement; +package org.hisp.dhis.jdbc; import java.util.Date; import org.hisp.dhis.program.AnalyticsPeriodBoundary; From 4038b0d88f7bdb7fea8c8cbb0269612a9cd87f0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sat, 30 Nov 2024 22:40:40 +0100 Subject: [PATCH 039/170] fix: Update code --- .../table/AbstractEventJdbcTableManager.java | 9 +++++---- .../table/JdbcEventAnalyticsTableManager.java | 18 +++++++++++++----- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index f00f880375ec..c7d3e6977a6d 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -32,6 +32,7 @@ import java.util.List; import java.util.Map; + import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.partition.PartitionManager; import org.hisp.dhis.analytics.table.model.AnalyticsTableColumn; @@ -224,20 +225,20 @@ protected void populateTableInternal(AnalyticsTablePartition partition, String f * * @param attribute the {@link TrackedEntityAttribute}. * @param columnExpression the column expression. - * @param dataClause the data type related clause like "NUMERIC". + * @param dataFilterClause the data filter clause. * @return a select statement. */ protected String getSelectSubquery( - TrackedEntityAttribute attribute, String columnExpression, String dataClause) { + TrackedEntityAttribute attribute, String columnExpression, String dataFilterClause) { return replaceQualify( """ (select ${columnExpression} from ${trackedentityattributevalue} \ where trackedentityid=en.trackedentityid \ - and trackedentityattributeid=${attributeId}${dataClause})\ + and trackedentityattributeid=${attributeId}${dataFilterClause})\ ${closingParentheses} as ${attributeUid}""", Map.of( "columnExpression", columnExpression, - "dataClause", dataClause, + "dataFilterClause", dataFilterClause, "attributeId", String.valueOf(attribute.getId()), "closingParentheses", getClosingParentheses(columnExpression), "attributeUid", quote(attribute.getUid()))); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 7735272ef20e..731207bb5376 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -51,7 +51,7 @@ import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; + import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.AnalyticsTableType; import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; @@ -92,6 +92,8 @@ import org.springframework.transaction.annotation.Transactional; import org.springframework.util.Assert; +import lombok.extern.slf4j.Slf4j; + /** * @author Lars Helge Overland */ @@ -629,14 +631,16 @@ private List getAttributeColumns(Program program) { private List getColumnForAttribute(TrackedEntityAttribute attribute) { List columns = new ArrayList<>(); + String columnName = String.format("%s.value", quote(attribute.getUid())); DataType dataType = getColumnType(attribute.getValueType(), isSpatialSupport()); - String selectExpression = getSelectExpressionForAttribute(attribute.getValueType(), "value"); - String dataExpression = getDataFilterClause(attribute); - String sql = getSelectSubquery(attribute, selectExpression, dataExpression); + String selectExpression = getSelectExpressionForAttribute(attribute.getValueType(), columnName); + String dataFilterClause = getDataFilterClause(attribute); + String sql = getSelectForInsert(attribute, selectExpression); + //String sql = getSelectSubquery(attribute, selectExpression, dataFilterClause); Skip skipIndex = skipIndex(attribute.getValueType(), attribute.hasOptionSet()); if (attribute.getValueType().isOrganisationUnit()) { - columns.addAll(getColumnsForOrgUnitTrackedEntityAttribute(attribute, dataExpression)); + columns.addAll(getColumnsForOrgUnitTrackedEntityAttribute(attribute, dataFilterClause)); } columns.add( @@ -762,6 +766,10 @@ private String getSelectForInsert( "uid", quote(dataElement.getUid()))); } + + private String getSelectForInsert(TrackedEntityAttribute attribute, String selectExpression) { + return String.format( "%s as %s", selectExpression, quote(attribute.getUid())); + } /** * Returns a list of columns. From b48b83c90e825b84b0e9fc73898d5d9ee9acb2bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 1 Dec 2024 11:33:21 +0100 Subject: [PATCH 040/170] fix: Update code --- .../table/AbstractEventJdbcTableManager.java | 1 - .../table/AbstractJdbcTableManager.java | 2 +- .../table/JdbcEventAnalyticsTableManager.java | 22 +++++++++++-------- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index c7d3e6977a6d..0a60a2d5042f 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -32,7 +32,6 @@ import java.util.List; import java.util.Map; - import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.partition.PartitionManager; import org.hisp.dhis.analytics.table.model.AnalyticsTableColumn; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java index d21a187ed63c..9969f14e6587 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java @@ -431,7 +431,7 @@ protected AnalyticsTable getLatestAnalyticsTable( protected void invokeTimeAndLog(String sql, String logPattern, Object... args) { Timer timer = new SystemTimer().start(); - log.debug("Populate table SQL: '{}'", sql); + log.info("Populate table SQL: '{}'", sql); jdbcTemplate.execute(sql); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 731207bb5376..6c270dd21f7d 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -51,7 +51,7 @@ import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; - +import lombok.extern.slf4j.Slf4j; import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.AnalyticsTableType; import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; @@ -92,8 +92,6 @@ import org.springframework.transaction.annotation.Transactional; import org.springframework.util.Assert; -import lombok.extern.slf4j.Slf4j; - /** * @author Lars Helge Overland */ @@ -386,7 +384,7 @@ and ev.status in (${exportableEventStatues}) \ private String getAttributeValueJoinClause(Program program) { String template = """ - left join ${trackedentityattributevalue} ${uid} \ + left join ${trackedentityattributevalue} as ${uid} \ on en.trackedentityid=${uid}.trackedentityid \ and ${uid}.trackedentityattributeid = ${id}\s"""; @@ -636,7 +634,6 @@ private List getColumnForAttribute(TrackedEntityAttribute String selectExpression = getSelectExpressionForAttribute(attribute.getValueType(), columnName); String dataFilterClause = getDataFilterClause(attribute); String sql = getSelectForInsert(attribute, selectExpression); - //String sql = getSelectSubquery(attribute, selectExpression, dataFilterClause); Skip skipIndex = skipIndex(attribute.getValueType(), attribute.hasOptionSet()); if (attribute.getValueType().isOrganisationUnit()) { @@ -740,12 +737,12 @@ private List getColumnsForOrgUnitTrackedEntityAttribute( } /** - * Creates a select statement for the given select expression. + * Returns a select statement. * * @param dataElement the data element to create the select statement for. * @param selectExpression the select expression. * @param dataFilterClause the data filter clause. - * @return A SQL select expression for the data element. + * @return a select expression. */ private String getSelectForInsert( DataElement dataElement, String selectExpression, String dataFilterClause) { @@ -766,9 +763,16 @@ private String getSelectForInsert( "uid", quote(dataElement.getUid()))); } - + + /** + * Returns a select statement. + * + * @param attribute the {@link TrackedEntityAttribute} + * @param selectExpression the select expression. + * @return a select expression. + */ private String getSelectForInsert(TrackedEntityAttribute attribute, String selectExpression) { - return String.format( "%s as %s", selectExpression, quote(attribute.getUid())); + return String.format("%s as %s", selectExpression, quote(attribute.getUid())); } /** From f9ed2a325e8dfa6f08ce88ed6122ac8bb43b7479 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 1 Dec 2024 13:23:06 +0100 Subject: [PATCH 041/170] fix: Update code --- .../analytics/table/JdbcEventAnalyticsTableManager.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 6c270dd21f7d..edfeec059c38 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -51,7 +51,7 @@ import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; + import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.AnalyticsTableType; import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; @@ -92,6 +92,8 @@ import org.springframework.transaction.annotation.Transactional; import org.springframework.util.Assert; +import lombok.extern.slf4j.Slf4j; + /** * @author Lars Helge Overland */ @@ -348,7 +350,7 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti inner join ${organisationunit} ou on ev.organisationunitid=ou.organisationunitid \ left join analytics_rs_orgunitstructure ous on ev.organisationunitid=ous.organisationunitid \ left join analytics_rs_organisationunitgroupsetstructure ougs on ev.organisationunitid=ougs.organisationunitid \ - and (cast(${eventDateMonth} as date)=ougs.startdate or ougs.startdate is null) \ + and (ougs.startdate is null or ps.monthstartdate=ougs.startdate) \ left join ${organisationunit} enrollmentou on en.organisationunitid=enrollmentou.organisationunitid \ inner join analytics_rs_categorystructure acs on ev.attributeoptioncomboid=acs.categoryoptioncomboid \ left join analytics_rs_dateperiodstructure dps on cast(${eventDateExpression} as date)=dps.dateperiod \ @@ -362,7 +364,6 @@ left join analytics_rs_dateperiodstructure dps on cast(${eventDateExpression} as and ev.status in (${exportableEventStatues}) \ and ev.deleted = false""", Map.of( - "eventDateMonth", sqlBuilder.dateTrunc("month", eventDateExpression), "eventDateExpression", eventDateExpression, "partitionClause", partitionClause, "attributeJoinClause", attributeJoinClause, From 9a3afb83bbfc8ebac59008140b2760fef9470099 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 1 Dec 2024 13:23:31 +0100 Subject: [PATCH 042/170] fix: Update code --- .../dhis/analytics/table/JdbcEventAnalyticsTableManager.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index edfeec059c38..857ac570b760 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -51,7 +51,7 @@ import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; - +import lombok.extern.slf4j.Slf4j; import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.AnalyticsTableType; import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; @@ -92,8 +92,6 @@ import org.springframework.transaction.annotation.Transactional; import org.springframework.util.Assert; -import lombok.extern.slf4j.Slf4j; - /** * @author Lars Helge Overland */ From 92be248698adf416f3674fd08cbce7d8a5d209a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 1 Dec 2024 15:40:58 +0100 Subject: [PATCH 043/170] fix: Update code --- .../table/JdbcEventAnalyticsTableManager.java | 4 ++-- .../resourcetable/table/DatePeriodResourceTable.java | 10 +++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 857ac570b760..568578945f4a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -346,12 +346,12 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti left join ${trackedentity} te on en.trackedentityid=te.trackedentityid and te.deleted = false \ left join ${organisationunit} registrationou on te.organisationunitid=registrationou.organisationunitid \ inner join ${organisationunit} ou on ev.organisationunitid=ou.organisationunitid \ + left join analytics_rs_dateperiodstructure dps on cast(${eventDateExpression} as date)=dps.dateperiod \ left join analytics_rs_orgunitstructure ous on ev.organisationunitid=ous.organisationunitid \ left join analytics_rs_organisationunitgroupsetstructure ougs on ev.organisationunitid=ougs.organisationunitid \ - and (ougs.startdate is null or ps.monthstartdate=ougs.startdate) \ + and (ougs.startdate is null or dps.monthstartdate=ougs.startdate) \ left join ${organisationunit} enrollmentou on en.organisationunitid=enrollmentou.organisationunitid \ inner join analytics_rs_categorystructure acs on ev.attributeoptioncomboid=acs.categoryoptioncomboid \ - left join analytics_rs_dateperiodstructure dps on cast(${eventDateExpression} as date)=dps.dateperiod \ ${attributeJoinClause}\ where ev.lastupdated < '${startTime}' ${partitionClause} \ and pr.programid=${programId} \ diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/table/DatePeriodResourceTable.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/table/DatePeriodResourceTable.java index ca86139beedf..444ca2dc4464 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/table/DatePeriodResourceTable.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/table/DatePeriodResourceTable.java @@ -49,6 +49,7 @@ import org.hisp.dhis.period.PeriodType; import org.hisp.dhis.resourcetable.ResourceTable; import org.hisp.dhis.resourcetable.ResourceTableType; +import org.hisp.dhis.util.DateUtils; /** * @author Lars Helge Overland @@ -75,7 +76,8 @@ private List getColumns() { List columns = Lists.newArrayList( new Column("dateperiod", DataType.DATE, Nullable.NOT_NULL), - new Column("year", DataType.INTEGER, Nullable.NOT_NULL)); + new Column("year", DataType.INTEGER, Nullable.NOT_NULL), + new Column("monthstartdate", DataType.DATE, Nullable.NOT_NULL)); for (PeriodType periodType : PeriodType.PERIOD_TYPES) { columns.add(new Column(periodType.getName().toLowerCase(), DataType.VARCHAR_50)); @@ -119,12 +121,14 @@ public Optional> getPopulateTempTableContent() { Calendar calendar = PeriodType.getCalendar(); for (Date day : days) { - List values = new ArrayList<>(); + final int year = PeriodType.getCalendar().fromIso(day).getYear(); + final Date monthStartDate = DateUtils.dateTruncMonth(day); - int year = PeriodType.getCalendar().fromIso(day).getYear(); + List values = new ArrayList<>(); values.add(day); values.add(year); + values.add(monthStartDate); for (PeriodType periodType : periodTypes) { values.add(periodType.createPeriod(day, calendar).getIsoDate()); From 31524cac022257ab15a847391b0a1d8abad18f52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 1 Dec 2024 19:53:29 +0100 Subject: [PATCH 044/170] fix: Update code --- .../dhis/analytics/table/JdbcEventAnalyticsTableManager.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 622aab8cfee0..c69715b7c7a7 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -736,8 +736,7 @@ private List getColumnsForOrgUnitTrackedEntityAttribute( } /** - * <<<<<<< HEAD Returns a select statement. ======= Retyrns a select statement for the given - * select expression. >>>>>>> master + * Returns a select statement for the given select expression. * * @param dataElement the data element to create the select statement for. * @param selectExpression the select expression. From 4259834a0526db714f2f6718f130d871e877e466 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 2 Dec 2024 14:55:20 +0100 Subject: [PATCH 045/170] fix: Update code --- .../table/JdbcEventAnalyticsTableManager.java | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index e8adcab5fa1b..8789dd8711b7 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -659,7 +659,7 @@ private List getColumnForAttributeWithLegendSet( } /** - * Retyrns a select statement for the given select expression. + * Returns a select statement for the given select expression. * * @param dataElement the data element to create the select statement for. * @param selectExpression the select expression. @@ -686,17 +686,6 @@ private String getSelectForInsert( quote(dataElement.getUid()))); } - /** - * Returns a select statement. - * - * @param attribute the {@link TrackedEntityAttribute} - * @param selectExpression the select expression. - * @return a select expression. - */ - private String getSelectForInsert(TrackedEntityAttribute attribute, String selectExpression) { - return String.format("%s as %s", selectExpression, quote(attribute.getUid())); - } - /** * Returns a list of columns. * From 25f4feb4323526bb09be7433cbce47630a9c673d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 2 Dec 2024 16:08:15 +0100 Subject: [PATCH 046/170] fix: Update code --- .../dhis/analytics/table/AbstractEventJdbcTableManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 5807a308ae4e..3d45e11185f5 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -261,7 +261,7 @@ protected List getColumnForAttribute(TrackedEntityAttribut Skip skipIndex = skipIndex(attribute.getValueType(), attribute.hasOptionSet()); if (attribute.getValueType().isOrganisationUnit()) { - columns.addAll(getColumnForOrgUnitTrackedEntityAttribute(attribute, dataFilterClause)); + columns.addAll(getColumnForOrgUnitAttribute(attribute, dataFilterClause)); } columns.add( @@ -283,7 +283,7 @@ protected List getColumnForAttribute(TrackedEntityAttribut * @param dataFilterClause the data filter clause. * @return a list of {@link AnalyticsTableColumn}. */ - private List getColumnForOrgUnitTrackedEntityAttribute( + private List getColumnForOrgUnitAttribute( TrackedEntityAttribute attribute, String dataFilterClause) { List columns = new ArrayList<>(); From df215eccd7acac21c2fd93fdd658d69d0b25d2ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 2 Dec 2024 16:10:03 +0100 Subject: [PATCH 047/170] fix: Update code --- .../table/JdbcEventAnalyticsTableManager.java | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 8789dd8711b7..16ff1804e26d 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -673,17 +673,14 @@ private String getSelectForInsert( ? "(select ${selectExpression} ${dataClause})${closingParentheses} as ${uid}" : "${selectExpression}${closingParentheses} as ${uid}"; + String s = getClosingParentheses(selectExpression); return replaceQualify( sqlTemplate, Map.of( - "selectExpression", - selectExpression, - "dataClause", - dataFilterClause, - "closingParentheses", - getClosingParentheses(selectExpression), - "uid", - quote(dataElement.getUid()))); + "selectExpression", selectExpression, + "dataClause", dataFilterClause, + "closingParentheses", s, + "uid", quote(dataElement.getUid()))); } /** From 83b4758bd9efd0f3dbc0e2b5d765cda7b5221095 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 2 Dec 2024 16:10:40 +0100 Subject: [PATCH 048/170] fix: Update code --- .../dhis/analytics/table/JdbcEventAnalyticsTableManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 16ff1804e26d..fa895ddb66a7 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -668,18 +668,18 @@ private List getColumnForAttributeWithLegendSet( */ private String getSelectForInsert( DataElement dataElement, String selectExpression, String dataFilterClause) { + String parentheses = getClosingParentheses(selectExpression); String sqlTemplate = dataElement.getValueType().isOrganisationUnit() ? "(select ${selectExpression} ${dataClause})${closingParentheses} as ${uid}" : "${selectExpression}${closingParentheses} as ${uid}"; - String s = getClosingParentheses(selectExpression); return replaceQualify( sqlTemplate, Map.of( "selectExpression", selectExpression, "dataClause", dataFilterClause, - "closingParentheses", s, + "closingParentheses", parentheses, "uid", quote(dataElement.getUid()))); } From 7a481d62b185207f926be6d2a66b5fb9e42f3680 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 2 Dec 2024 16:13:13 +0100 Subject: [PATCH 049/170] fix: Update code --- .../analytics/table/JdbcEventAnalyticsTableManager.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index fa895ddb66a7..04102a4d9d8f 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -758,15 +758,14 @@ private List getDataYears( Program program, Integer firstDataYear, Integer lastDataYear) { + String fromDate = toMediumDate(params.getFromDate()); String fromDateClause = params.getFromDate() != null ? replace( "and (${eventDateExpression}) >= '${fromDate}'", Map.of( - "eventDateExpression", - eventDateExpression, - "fromDate", - toMediumDate(params.getFromDate()))) + "eventDateExpression", eventDateExpression, + "fromDate", fromDate)) : EMPTY; String sql = From bca8df519e9503ed08e8b7a355bc94f7201af75c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 2 Dec 2024 16:40:43 +0100 Subject: [PATCH 050/170] fix: Update code --- .../table/AbstractEventJdbcTableManager.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 3d45e11185f5..1350cc99b9ae 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -34,7 +34,6 @@ import static org.hisp.dhis.db.model.DataType.GEOMETRY; import static org.hisp.dhis.db.model.DataType.TEXT; import static org.hisp.dhis.system.util.MathUtils.NUMERIC_LENIENT_REGEXP; - import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -172,9 +171,9 @@ private String getSelectExpression(ValueType valueType, String columnExpression, } else if (valueType.isOrganisationUnit()) { String ouClause = isTea - ? "ou.uid from ${organisationunit} ou where ou.uid = (select ${columnName}" - : "ou.uid from ${organisationunit} ou where ou.uid = ${columnName}"; - return replaceQualify(ouClause, Map.of("columnName", columnExpression)); + ? "ou.uid from ${organisationunit} ou where ou.uid = (select ${columnExpression}" + : "ou.uid from ${organisationunit} ou where ou.uid = ${columnExpression}"; + return replaceQualify(ouClause, Map.of("columnExpression", columnExpression)); } else { return columnExpression; } @@ -317,6 +316,13 @@ private List getColumnForOrgUnitAttribute( return columns; } + + private String getSelectForInsert(TrackedEntityAttribute attribute, String selectExpression, String dataFilterClause) { + return + String.format( + """ + %s.%s as %s, """); + } /** * The select subquery statement. From fdc55784bc981ca47b31f8ab9a56918ee5c4fd3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 2 Dec 2024 16:40:52 +0100 Subject: [PATCH 051/170] fix: Update code --- .../analytics/table/AbstractEventJdbcTableManager.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 1350cc99b9ae..c1f8c34172fd 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -34,6 +34,7 @@ import static org.hisp.dhis.db.model.DataType.GEOMETRY; import static org.hisp.dhis.db.model.DataType.TEXT; import static org.hisp.dhis.system.util.MathUtils.NUMERIC_LENIENT_REGEXP; + import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -316,10 +317,10 @@ private List getColumnForOrgUnitAttribute( return columns; } - - private String getSelectForInsert(TrackedEntityAttribute attribute, String selectExpression, String dataFilterClause) { - return - String.format( + + private String getSelectForInsert( + TrackedEntityAttribute attribute, String selectExpression, String dataFilterClause) { + return String.format( """ %s.%s as %s, """); } From 663386103850260f51ce417c5991002c06276a11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 09:45:35 +0100 Subject: [PATCH 052/170] Revert "feat: skipAnalytics and ConfidentialFlag only for QUERY Analytics [DHIS2-18259] (#19350)" This reverts commit dc0fa5ef4343ad2eba3ddd8c059735edbf7b8d4a. --- .../dhis/common/DimensionalItemObject.java | 4 - .../org/hisp/dhis/feedback/ErrorCode.java | 3 - .../java/org/hisp/dhis/program/Program.java | 34 +++- .../org/hisp/dhis/program/ProgramTest.java | 5 + .../analytics/event/EventQueryParams.java | 5 - ...tEnrollmentAnalyticsDimensionsService.java | 41 ++--- ...efaultEventAnalyticsDimensionsService.java | 15 +- .../data/DefaultEventDataQueryService.java | 69 -------- .../table/AbstractEventJdbcTableManager.java | 2 +- .../JdbcEnrollmentAnalyticsTableManager.java | 2 +- .../table/JdbcEventAnalyticsTableManager.java | 8 +- ...dbcTrackedEntityAnalyticsTableManager.java | 12 +- .../DefaultEventDataQueryServiceTest.java | 151 ------------------ ...rackedEntityAnalyticsTableManagerTest.java | 7 +- .../analytics/AnalyticsDimensionsTest.java | 2 +- 15 files changed, 89 insertions(+), 271 deletions(-) delete mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/DefaultEventDataQueryServiceTest.java diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalItemObject.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalItemObject.java index 4134b7314387..c12c4e954df7 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalItemObject.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalItemObject.java @@ -87,8 +87,4 @@ public interface DimensionalItemObject extends NameableObject { default int getPeriodOffset() { return (getQueryMods() != null) ? getQueryMods().getPeriodOffset() : 0; } - - default boolean isOfType(DimensionItemType type) { - return type == getDimensionItemType(); - } } diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/feedback/ErrorCode.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/feedback/ErrorCode.java index 9dcf3008b561..1c1cf44d0afe 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/feedback/ErrorCode.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/feedback/ErrorCode.java @@ -505,9 +505,6 @@ public enum ErrorCode { E7236("Program stage '{0}' is not associated to program '{0}'"), E7237("Sorting must have a valid dimension and a direction"), E7238("Sorting dimension ‘{0}’ is not a column"), - E7239( - "Tracked Entity Attributes marked as 'confidential' can only be used in aggregate analytics: `{0}`"), - E7240("Data Elements marked as 'skipAnalytics' can only be used in aggregate analytics: `{0}`"), /* TE analytics */ E7250("Dimension is not a fully qualified: `{0}`"), diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/Program.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/Program.java index 8d9b8da95009..d6ca3575963d 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/Program.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/Program.java @@ -37,9 +37,11 @@ import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; import java.util.ArrayList; +import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; import org.hisp.dhis.category.CategoryCombo; @@ -260,12 +262,26 @@ public Set getDataElements() { return programStages.stream().flatMap(ps -> ps.getDataElements().stream()).collect(toSet()); } + /** + * Returns all data elements which are part of the stages of this program and is not skipped in + * analytics. + */ + public Set getAnalyticsDataElements() { + return programStages.stream() + .map(ProgramStage::getProgramStageDataElements) + .flatMap(Collection::stream) + .filter(Objects::nonNull) + .filter(psde -> !psde.getSkipAnalytics()) + .map(ProgramStageDataElement::getDataElement) + .collect(toSet()); + } + /** * Returns data elements which are part of the stages of this program which have a legend set and * is of numeric value type. */ - public Set getDataElementsWithLegendSet() { - return getDataElements().stream() + public Set getAnalyticsDataElementsWithLegendSet() { + return getAnalyticsDataElements().stream() .filter(de -> de.hasLegendSet() && de.isNumericType()) .collect(toSet()); } @@ -280,13 +296,23 @@ public List getTrackedEntityAttributes() { .collect(Collectors.toList()); } + /** + * Returns non-confidential TrackedEntityAttributes from ProgramTrackedEntityAttributes. Use + * getAttributes() to access the persisted attribute list. + */ + public List getNonConfidentialTrackedEntityAttributes() { + return getTrackedEntityAttributes().stream() + .filter(a -> !a.isConfidentialBool()) + .collect(Collectors.toList()); + } + /** * Returns TrackedEntityAttributes from ProgramTrackedEntityAttributes which have a legend set and * is of numeric value type. */ - public List getTrackedEntityAttributesWithLegendSet() { + public List getNonConfidentialTrackedEntityAttributesWithLegendSet() { return getTrackedEntityAttributes().stream() - .filter(a -> a.hasLegendSet() && a.isNumericType()) + .filter(a -> !a.isConfidentialBool() && a.hasLegendSet() && a.isNumericType()) .collect(Collectors.toList()); } diff --git a/dhis-2/dhis-api/src/test/java/org/hisp/dhis/program/ProgramTest.java b/dhis-2/dhis-api/src/test/java/org/hisp/dhis/program/ProgramTest.java index 7f7cbff8216b..3555686ada7e 100644 --- a/dhis-2/dhis-api/src/test/java/org/hisp/dhis/program/ProgramTest.java +++ b/dhis-2/dhis-api/src/test/java/org/hisp/dhis/program/ProgramTest.java @@ -85,6 +85,7 @@ void testGetAnalyticsDataElements() { assertEquals(2, prA.getDataElements().size()); assertTrue(prA.getDataElements().contains(deA)); assertTrue(prA.getDataElements().contains(deB)); + assertEquals(1, prA.getAnalyticsDataElements().size()); assertTrue(prA.getDataElements().contains(deA)); } @@ -105,6 +106,7 @@ void testCopyOfWithPropertyValuesSet() { // check equal assertEquals(original.getAccess(), copy.getAccess()); assertEquals(original.getAccessLevel(), copy.getAccessLevel()); + assertEquals(original.getAnalyticsDataElements(), copy.getAnalyticsDataElements()); assertEquals(original.getCategoryCombo(), copy.getCategoryCombo()); assertEquals(original.getCompleteEventsExpiryDays(), copy.getCompleteEventsExpiryDays()); assertEquals(original.getDataElements(), copy.getDataElements()); @@ -178,7 +180,10 @@ void testCopyOfWithNulls() { assertEquals("copynull", copy.getName()); assertEquals(original.getAccessLevel(), copy.getAccessLevel()); assertEquals(original.getDescription(), copy.getDescription()); + assertTrue(copy.getAnalyticsDataElements().isEmpty()); assertTrue(copy.getDataElements().isEmpty()); + assertTrue(copy.getNonConfidentialTrackedEntityAttributes().isEmpty()); + assertTrue(copy.getNonConfidentialTrackedEntityAttributesWithLegendSet().isEmpty()); assertTrue(copy.getNotificationTemplates().isEmpty()); assertTrue(copy.getOrganisationUnits().isEmpty()); assertTrue(copy.getProgramAttributes().isEmpty()); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java index 7848156e434d..4d6f0683f691 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java @@ -37,7 +37,6 @@ import static org.hisp.dhis.common.DimensionalObject.PERIOD_DIM_ID; import static org.hisp.dhis.common.DimensionalObjectUtils.asList; import static org.hisp.dhis.common.DimensionalObjectUtils.asTypedList; -import static org.hisp.dhis.common.RequestTypeAware.EndpointAction.AGGREGATE; import static org.hisp.dhis.common.RequestTypeAware.EndpointAction.QUERY; import com.google.common.base.MoreObjects; @@ -999,10 +998,6 @@ public boolean isRowContext() { return rowContext; } - public boolean includeConfidentialOrSkipAnalyticsItems() { - return endpointAction == AGGREGATE; - } - // ------------------------------------------------------------------------- // Builder of immutable instances // ------------------------------------------------------------------------- diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEnrollmentAnalyticsDimensionsService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEnrollmentAnalyticsDimensionsService.java index c13b40d3d4df..2894d3771eb6 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEnrollmentAnalyticsDimensionsService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEnrollmentAnalyticsDimensionsService.java @@ -31,26 +31,22 @@ import static org.hisp.dhis.analytics.common.DimensionsServiceCommon.OperationType.QUERY; import static org.hisp.dhis.analytics.common.DimensionsServiceCommon.collectDimensions; import static org.hisp.dhis.analytics.common.DimensionsServiceCommon.filterByValueType; -import static org.hisp.dhis.analytics.event.data.DefaultEventAnalyticsDimensionsService.getTeasIfRegistration; import static org.hisp.dhis.common.PrefixedDimensions.ofItemsWithProgram; import static org.hisp.dhis.common.PrefixedDimensions.ofProgramStageDataElements; import java.util.Collection; import java.util.List; import java.util.Optional; -import java.util.Set; -import java.util.function.Predicate; import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; import org.hisp.dhis.analytics.common.DimensionsServiceCommon; -import org.hisp.dhis.analytics.common.DimensionsServiceCommon.OperationType; import org.hisp.dhis.analytics.event.EnrollmentAnalyticsDimensionsService; import org.hisp.dhis.common.PrefixedDimension; import org.hisp.dhis.program.Program; import org.hisp.dhis.program.ProgramService; import org.hisp.dhis.program.ProgramStage; -import org.hisp.dhis.program.ProgramStageDataElement; import org.hisp.dhis.security.acl.AclService; +import org.hisp.dhis.trackedentity.TrackedEntityAttribute; import org.hisp.dhis.user.CurrentUserUtil; import org.hisp.dhis.user.UserDetails; import org.springframework.stereotype.Service; @@ -81,7 +77,9 @@ public List getQueryDimensionsByProgramId(String programId) { .collect(Collectors.toSet())), getProgramStageDataElements(QUERY, program), filterByValueType( - QUERY, ofItemsWithProgram(program, getTeasIfRegistration(program)))))) + QUERY, + ofItemsWithProgram( + program, getTeasIfRegistrationAndNotConfidential(program)))))) .orElse(List.of()); } @@ -89,22 +87,14 @@ private Collection getProgramStageDataElements( DimensionsServiceCommon.OperationType operationType, Program program) { return program.getProgramStages().stream() .map(ProgramStage::getProgramStageDataElements) - .map(psdes -> excludeIfSkipAnalytics(operationType, psdes)) - .map(psdes -> filterByValueType(operationType, ofProgramStageDataElements(psdes))) + .map( + programStageDataElements -> + filterByValueType( + operationType, ofProgramStageDataElements(programStageDataElements))) .flatMap(Collection::stream) .collect(Collectors.toList()); } - private Set excludeIfSkipAnalytics( - OperationType operationType, Set programStageDataElements) { - if (operationType == QUERY) { - return programStageDataElements.stream() - .filter(Predicate.not(ProgramStageDataElement::getSkipAnalytics)) - .collect(Collectors.toSet()); - } - return programStageDataElements; - } - @Override public List getAggregateDimensionsByProgramStageId(String programId) { return Optional.of(programId) @@ -119,4 +109,19 @@ public List getAggregateDimensionsByProgramStageId(String pro ofItemsWithProgram(program, program.getTrackedEntityAttributes()))))) .orElse(List.of()); } + + private Collection getTeasIfRegistrationAndNotConfidential( + Program program) { + return Optional.of(program) + .filter(Program::isRegistration) + .map(Program::getTrackedEntityAttributes) + .orElse(List.of()) + .stream() + .filter(this::isNotConfidential) + .collect(Collectors.toList()); + } + + private boolean isNotConfidential(TrackedEntityAttribute trackedEntityAttribute) { + return !trackedEntityAttribute.isConfidentialBool(); + } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsDimensionsService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsDimensionsService.java index a67b341b61df..76086a2c4b27 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsDimensionsService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsDimensionsService.java @@ -135,7 +135,9 @@ private List dimensions(ProgramStage programStage) { .filter(pi -> aclService.canRead(currentUserDetails, pi)) .collect(Collectors.toSet())), filterByValueType(QUERY, ofDataElements(programStage)), - filterByValueType(QUERY, ofItemsWithProgram(p, getTeasIfRegistration(p))), + filterByValueType( + QUERY, + ofItemsWithProgram(p, getTeasIfRegistrationAndNotConfidential(p))), ofItemsWithProgram(p, getCategories(p)), ofItemsWithProgram(p, getAttributeCategoryOptionGroupSetsIfNeeded(p))))) .orElse(List.of()); @@ -185,10 +187,17 @@ private List getCategories(Program program) { .orElse(List.of()); } - static List getTeasIfRegistration(Program program) { + private List getTeasIfRegistrationAndNotConfidential(Program program) { return Optional.of(program) .filter(Program::isRegistration) .map(Program::getTrackedEntityAttributes) - .orElse(List.of()); + .orElse(List.of()) + .stream() + .filter(this::isNotConfidential) + .collect(Collectors.toList()); + } + + private boolean isNotConfidential(TrackedEntityAttribute trackedEntityAttribute) { + return !trackedEntityAttribute.isConfidentialBool(); } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventDataQueryService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventDataQueryService.java index 3d20c79d5a61..b08af886985a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventDataQueryService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventDataQueryService.java @@ -36,8 +36,6 @@ import static org.hisp.dhis.analytics.event.data.DefaultEventDataQueryService.SortableItems.translateItemIfNecessary; import static org.hisp.dhis.analytics.util.AnalyticsUtils.illegalQueryExSupplier; import static org.hisp.dhis.analytics.util.AnalyticsUtils.throwIllegalQueryEx; -import static org.hisp.dhis.common.DimensionItemType.DATA_ELEMENT; -import static org.hisp.dhis.common.DimensionItemType.PROGRAM_ATTRIBUTE; import static org.hisp.dhis.common.DimensionalObject.DIMENSION_NAME_SEP; import static org.hisp.dhis.common.DimensionalObject.PERIOD_DIM_ID; import static org.hisp.dhis.common.DimensionalObjectUtils.getDimensionFromParam; @@ -54,7 +52,6 @@ import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; -import java.util.stream.Stream; import lombok.Getter; import lombok.RequiredArgsConstructor; import org.apache.commons.lang3.StringUtils; @@ -70,7 +67,6 @@ import org.hisp.dhis.analytics.table.EnrollmentAnalyticsColumnName; import org.hisp.dhis.analytics.table.EventAnalyticsColumnName; import org.hisp.dhis.common.BaseDimensionalItemObject; -import org.hisp.dhis.common.BaseIdentifiableObject; import org.hisp.dhis.common.DimensionalItemObject; import org.hisp.dhis.common.DimensionalObject; import org.hisp.dhis.common.EventAnalyticalObject; @@ -94,7 +90,6 @@ import org.hisp.dhis.program.Program; import org.hisp.dhis.program.ProgramService; import org.hisp.dhis.program.ProgramStage; -import org.hisp.dhis.program.ProgramStageDataElement; import org.hisp.dhis.program.ProgramStageService; import org.hisp.dhis.setting.UserSettings; import org.hisp.dhis.trackedentity.TrackedEntityAttribute; @@ -228,73 +223,9 @@ public EventQueryParams getFromRequest(EventDataQueryRequest request, boolean an eventQueryParams = builder.build(); } - validateQueryParamsForConfidentialAndSkipAnalytics(eventQueryParams); - return eventQueryParams; } - static void validateQueryParamsForConfidentialAndSkipAnalytics( - EventQueryParams eventQueryParams) { - if (eventQueryParams.includeConfidentialOrSkipAnalyticsItems()) { - return; - } - Set confidentialAttributes = - Stream.concat( - eventQueryParams.getItems().stream(), eventQueryParams.getItemFilters().stream()) - .map(QueryItem::getItem) - .filter(Objects::nonNull) - .filter(dimObj -> dimObj.isOfType(PROGRAM_ATTRIBUTE)) - .map(TrackedEntityAttribute.class::cast) - .filter(TrackedEntityAttribute::isConfidentialBool) - .collect(Collectors.toSet()); - - if (!confidentialAttributes.isEmpty()) { - throw new IllegalQueryException( - new ErrorMessage( - ErrorCode.E7239, - confidentialAttributes.stream() - .map(TrackedEntityAttribute::getUid) - .collect(Collectors.joining(", ")))); - } - - Set skipAnalyticsDataElements = - Stream.concat( - eventQueryParams.getItems().stream(), eventQueryParams.getItemFilters().stream()) - .filter(item -> item.getItem().isOfType(DATA_ELEMENT)) - .filter(DefaultEventDataQueryService::isSkipAnalytics) - .map(item -> (DataElement) item.getItem()) - .collect(Collectors.toSet()); - - if (!skipAnalyticsDataElements.isEmpty()) { - throw new IllegalQueryException( - new ErrorMessage( - ErrorCode.E7240, - skipAnalyticsDataElements.stream() - .map(DataElement::getUid) - .collect(Collectors.joining(", ")))); - } - } - - /** - * Checks if the data element is marked as skip analytics by looking at the program stage data - * elements associated with the program stage. If any of the program stage data elements has the - * skip analytics flag set to true, the data element in it is considered to be skipAnalytics. - * - * @param item the query item - * @return true if the data element is marked as skip analytics, false otherwise - */ - static boolean isSkipAnalytics(QueryItem item) { - return Optional.of(item) - .map(QueryItem::getProgramStage) - .map(ProgramStage::getProgramStageDataElements) - .orElse(Set.of()) - .stream() - .filter(ProgramStageDataElement::getSkipAnalytics) - .map(ProgramStageDataElement::getDataElement) - .map(BaseIdentifiableObject::getUid) - .anyMatch(uid -> uid.equals(item.getItem().getUid())); - } - private boolean hasPeriodDimension(EventQueryParams eventQueryParams) { return Objects.nonNull(getPeriodDimension(eventQueryParams)); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 82270c5d2587..60d6c433ad3a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -214,7 +214,7 @@ protected void populateTableInternal(AnalyticsTablePartition partition, String f * Returns a list of columns based on the given attribute. * * @param attribute the {@link TrackedEntityAttribute}. - * @return a list of {@link AnalyticsTableColumn}. + * @return a list of {@link AnaylyticsTableColumn}. */ protected List getColumnForAttribute(TrackedEntityAttribute attribute) { List columns = new ArrayList<>(); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java index 59bdf500f0a5..d18fbdffd80b 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java @@ -187,7 +187,7 @@ private List getColumns(Program program) { * @return a list of {@link AnalyticsTableColumn}. */ private List getTrackedEntityAttributeColumns(Program program) { - return program.getTrackedEntityAttributes().stream() + return program.getNonConfidentialTrackedEntityAttributes().stream() .map(this::getColumnForAttribute) .flatMap(Collection::stream) .toList(); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 2da49a036477..74d53d5adb60 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -497,12 +497,12 @@ private List getAttributeCategoryColumns(Program program) private List getDataElementColumns(Program program) { List columns = new ArrayList<>(); columns.addAll( - program.getDataElements().stream() + program.getAnalyticsDataElements().stream() .map(de -> getColumnForDataElement(de, false)) .flatMap(Collection::stream) .toList()); columns.addAll( - program.getDataElementsWithLegendSet().stream() + program.getAnalyticsDataElementsWithLegendSet().stream() .map(de -> getColumnForDataElement(de, true)) .flatMap(Collection::stream) .toList()); @@ -603,12 +603,12 @@ private List getColumnForOrgUnitDataElement( private List getAttributeColumns(Program program) { List columns = new ArrayList<>(); columns.addAll( - program.getTrackedEntityAttributes().stream() + program.getNonConfidentialTrackedEntityAttributes().stream() .map(this::getColumnForAttribute) .flatMap(Collection::stream) .toList()); columns.addAll( - program.getTrackedEntityAttributesWithLegendSet().stream() + program.getNonConfidentialTrackedEntityAttributesWithLegendSet().stream() .map(this::getColumnForAttributeWithLegendSet) .flatMap(Collection::stream) .toList()); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java index e2d0bdb9e5ac..4a483d156d37 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java @@ -88,7 +88,8 @@ public class JdbcTrackedEntityAnalyticsTableManager extends AbstractJdbcTableManager { private static final String PROGRAMS_BY_TET_KEY = "programsByTetUid"; - private static final String ALL_TET_ATTRIBUTES = "allTetAttributes"; + private static final String ALL_NON_CONFIDENTIAL_TET_ATTRIBUTES = + "allNonConfidentialTetAttributes"; private final TrackedEntityTypeService trackedEntityTypeService; @@ -199,9 +200,12 @@ private List getColumns( .build())); List trackedEntityAttributes = - getAllTrackedEntityAttributes(trackedEntityType, programsByTetUid).toList(); + getAllTrackedEntityAttributes(trackedEntityType, programsByTetUid) + .filter(tea -> !tea.isConfidentialBool()) + .toList(); - params.addExtraParam(trackedEntityType.getUid(), ALL_TET_ATTRIBUTES, trackedEntityAttributes); + params.addExtraParam( + trackedEntityType.getUid(), ALL_NON_CONFIDENTIAL_TET_ATTRIBUTES, trackedEntityAttributes); columns.addAll( trackedEntityAttributes.stream() @@ -351,7 +355,7 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti Map.of())); ((List) - params.getExtraParam(trackedEntityType.getUid(), ALL_TET_ATTRIBUTES)) + params.getExtraParam(trackedEntityType.getUid(), ALL_NON_CONFIDENTIAL_TET_ATTRIBUTES)) .forEach( tea -> sql.append( diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/DefaultEventDataQueryServiceTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/DefaultEventDataQueryServiceTest.java deleted file mode 100644 index 0a54a25dc6aa..000000000000 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/DefaultEventDataQueryServiceTest.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright (c) 2004-2022, University of Oslo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * Neither the name of the HISP project nor the names of its contributors may - * be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package org.hisp.dhis.analytics.event.data; - -import static org.hisp.dhis.analytics.event.data.DefaultEventDataQueryService.validateQueryParamsForConfidentialAndSkipAnalytics; -import static org.hisp.dhis.common.DimensionItemType.DATA_ELEMENT; -import static org.hisp.dhis.common.DimensionItemType.PROGRAM_ATTRIBUTE; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.util.List; -import java.util.Set; -import java.util.function.Consumer; -import org.hisp.dhis.analytics.event.EventQueryParams; -import org.hisp.dhis.common.DimensionItemType; -import org.hisp.dhis.common.DimensionalItemObject; -import org.hisp.dhis.common.IllegalQueryException; -import org.hisp.dhis.common.QueryItem; -import org.hisp.dhis.dataelement.DataElement; -import org.hisp.dhis.program.ProgramStage; -import org.hisp.dhis.program.ProgramStageDataElement; -import org.hisp.dhis.trackedentity.TrackedEntityAttribute; -import org.junit.jupiter.api.Test; - -class DefaultEventDataQueryServiceTest { - - private EventQueryParams mockEventQueryParams( - boolean includeConfidentialOrSkipAnalyticsItems, List queryItems) { - EventQueryParams mock = mock(EventQueryParams.class); - when(mock.includeConfidentialOrSkipAnalyticsItems()) - .thenReturn(includeConfidentialOrSkipAnalyticsItems); - when(mock.getItems()).thenReturn(queryItems); - when(mock.getItemFilters()).thenReturn(List.of()); - return mock; - } - - private QueryItem mockDimensionalItemObject( - Class clazz, DimensionItemType type, List> behavious) { - T mock = mock(clazz); - when(mock.isOfType(type)).thenReturn(true); - QueryItem queryItem = mock(QueryItem.class); - when(queryItem.getItem()).thenReturn(mock); - behavious.forEach(c -> c.accept(mock)); - return queryItem; - } - - @Test - void testAggregateDontThrowExceptionForConfidential() { - QueryItem queryItem = - mockDimensionalItemObject( - TrackedEntityAttribute.class, - PROGRAM_ATTRIBUTE, - List.of(t -> when(t.isConfidentialBool()).thenReturn(true))); - EventQueryParams eventQueryParams = mockEventQueryParams(true, List.of(queryItem)); - assertDoesNotThrow(() -> validateQueryParamsForConfidentialAndSkipAnalytics(eventQueryParams)); - } - - @Test - void testAggregateDontThrowExceptionForSkipAnalytics() { - final String dataElementUid = "dataElementUid"; - - ProgramStage programStage = mock(ProgramStage.class); - ProgramStageDataElement programStageDataElement = mock(ProgramStageDataElement.class); - - when(programStage.getProgramStageDataElements()).thenReturn(Set.of(programStageDataElement)); - when(programStageDataElement.getSkipAnalytics()).thenReturn(true); - - DataElement dataElement = mock(DataElement.class); - when(dataElement.getUid()).thenReturn(dataElementUid); - when(programStageDataElement.getDataElement()).thenReturn(dataElement); - - QueryItem queryItem = - mockDimensionalItemObject( - DataElement.class, - DATA_ELEMENT, - List.of(t -> when(t.getUid()).thenReturn(dataElementUid))); - when(queryItem.getProgramStage()).thenReturn(programStage); - - EventQueryParams eventQueryParams = mockEventQueryParams(true, List.of(queryItem)); - - assertDoesNotThrow(() -> validateQueryParamsForConfidentialAndSkipAnalytics(eventQueryParams)); - } - - @Test - void testQueryThrowsForConfidential() { - QueryItem queryItem = - mockDimensionalItemObject( - TrackedEntityAttribute.class, - PROGRAM_ATTRIBUTE, - List.of(t -> when(t.isConfidentialBool()).thenReturn(true))); - EventQueryParams eventQueryParams = mockEventQueryParams(false, List.of(queryItem)); - assertThrows( - IllegalQueryException.class, - () -> validateQueryParamsForConfidentialAndSkipAnalytics(eventQueryParams)); - } - - @Test - void testQueryThrowsForSkipAnalytics() { - final String dataElementUid = "dataElementUid"; - - ProgramStage programStage = mock(ProgramStage.class); - ProgramStageDataElement programStageDataElement = mock(ProgramStageDataElement.class); - - when(programStage.getProgramStageDataElements()).thenReturn(Set.of(programStageDataElement)); - when(programStageDataElement.getSkipAnalytics()).thenReturn(true); - - DataElement dataElement = mock(DataElement.class); - when(dataElement.getUid()).thenReturn(dataElementUid); - when(programStageDataElement.getDataElement()).thenReturn(dataElement); - - QueryItem queryItem = - mockDimensionalItemObject( - DataElement.class, - DATA_ELEMENT, - List.of(t -> when(t.getUid()).thenReturn(dataElementUid))); - when(queryItem.getProgramStage()).thenReturn(programStage); - - EventQueryParams eventQueryParams = mockEventQueryParams(false, List.of(queryItem)); - - assertThrows( - IllegalQueryException.class, - () -> validateQueryParamsForConfidentialAndSkipAnalytics(eventQueryParams)); - } -} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManagerTest.java index b80504d26b7e..0d537f2e63da 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManagerTest.java @@ -28,6 +28,7 @@ package org.hisp.dhis.analytics.table; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; @@ -143,13 +144,13 @@ void verifyNonConfidentialTeasAreSkipped() { AnalyticsTable analyticsTable = analyticsTables.get(0); assertContainsNonConfidentialTeaColumns(analyticsTable); - assertContainsConfidentialTeaColumns(analyticsTable); + assertDoesntContainConfidentialTeaColumns(analyticsTable); } - private void assertContainsConfidentialTeaColumns(AnalyticsTable analyticsTable) { + private void assertDoesntContainConfidentialTeaColumns(AnalyticsTable analyticsTable) { List columns = analyticsTable.getColumns(); - assertTrue(columns.stream().map(Column::getName).anyMatch("confidentialTeaUid"::equals)); + assertFalse(columns.stream().map(Column::getName).anyMatch("confidentialTeaUid"::equals)); } private void assertContainsNonConfidentialTeaColumns(AnalyticsTable analyticsTable) { diff --git a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/analytics/AnalyticsDimensionsTest.java b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/analytics/AnalyticsDimensionsTest.java index 821adf4c0230..08811f890abf 100644 --- a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/analytics/AnalyticsDimensionsTest.java +++ b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/analytics/AnalyticsDimensionsTest.java @@ -183,7 +183,7 @@ public void shouldOnlyReturnConfidentialAttributeInAggregateDimensions() { .query() .getDimensionsByDimensionType(trackerProgram.getUid(), "PROGRAM_ATTRIBUTE") .validate() - .body("dimensions.uid", CoreMatchers.hasItem(confidentialAttribute)); + .body("dimensions.uid", not(CoreMatchers.hasItem(confidentialAttribute))); analyticsEnrollmentsActions .aggregate() From 693ac89091fa07b152bfbbbfa1d4d82a13cfcfb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 09:47:25 +0100 Subject: [PATCH 053/170] fix: Update code --- .../dhis/analytics/table/JdbcEventAnalyticsTableManager.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 74d53d5adb60..9d064681cf64 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -383,8 +383,7 @@ private String getAttributeValueJoinClause(Program program) { on en.trackedentityid=${uid}.trackedentityid \ and ${uid}.trackedentityattributeid = ${id}\s"""; - // TO DO non confidential? - return program.getTrackedEntityAttributes().stream() + return program.getNonConfidentialTrackedEntityAttributes().stream() .map(attribute -> replaceQualify(template, toVariableMap(attribute))) .collect(Collectors.joining()); } From 7b310eba19f76732af4b7e5532a2bf8ad1a66118 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 10:55:06 +0100 Subject: [PATCH 054/170] fix: Update code --- .../dhis/analytics/table/JdbcEventAnalyticsTableManager.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 9d064681cf64..f7cf760b316b 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -657,8 +657,7 @@ private List getColumnForAttributeWithLegendSet( } /** - * <<<<<<< HEAD Returns a select statement for the given select expression. ======= Returns a - * select statement for the given data element with value type org unit. >>>>>>> master + * Returns a select statement for the given select expression. * * @param dataElement the data element to create the select statement for. * @param selectExpression the select expression. From be271e6aa2995c8e29df261759a7b624babdbf5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 10:55:40 +0100 Subject: [PATCH 055/170] fix: Update code --- .../analytics/table/AbstractEventJdbcTableManager.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 60d6c433ad3a..9c9e893efd0a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -283,13 +283,6 @@ private List getColumnForOrgUnitAttribute( return columns; } - private String getSelectForInsert( - TrackedEntityAttribute attribute, String selectExpression, String dataFilterClause) { - return String.format( - """ - %s.%s as %s, """); - } - /** * The select subquery statement. * From e178384f2fe7117ca49e83872000eacbc454bc2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 10:58:51 +0100 Subject: [PATCH 056/170] fix: Update code --- .../dhis/analytics/table/AbstractEventJdbcTableManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 9c9e893efd0a..f309a62dcc0b 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -118,7 +118,6 @@ protected Skip skipIndex(ValueType valueType, boolean hasOptionSet) { /** * Returns a select expression, potentially with a cast statement, based on the given value type. - * Handles data element and tracked entity attribute select expressions. * * @param valueType the {@link ValueType} to represent as database column type. * @param columnExpression the expression or name of the column to be selected. @@ -138,7 +137,8 @@ protected String getSelectExpression(ValueType valueType, String columnExpressio } else if (valueType.isGeo() && isSpatialSupport()) { return String.format( """ - ST_GeomFromGeoJSON('{"type":"Point", "coordinates":' || (%s) || ', "crs":{"type":"name", "properties":{"name":"EPSG:4326"}}}')""", + ST_GeomFromGeoJSON('{"type":"Point", "coordinates":' || (%s) || \ + ', "crs":{"type":"name", "properties":{"name":"EPSG:4326"}}}')""", columnExpression); } else { return columnExpression; From d3c19579b08e4a2dc21b75ea4864b3bfe53e15b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 11:05:13 +0100 Subject: [PATCH 057/170] fix: Update code --- .../dhis/analytics/table/AbstractEventJdbcTableManager.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index f309a62dcc0b..8cf2e01dee72 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -219,10 +219,10 @@ protected void populateTableInternal(AnalyticsTablePartition partition, String f protected List getColumnForAttribute(TrackedEntityAttribute attribute) { List columns = new ArrayList<>(); + String valueColumn = String.format("%s.%s", quote(attribute.getUid()), "value"); DataType dataType = getColumnType(attribute.getValueType(), isSpatialSupport()); - String selectExpression = getSelectExpression(attribute.getValueType(), "value"); + String selectExpression = getSelectExpression(attribute.getValueType(), valueColumn); String dataFilterClause = getDataFilterClause(attribute); - String sql = getSelectSubquery(attribute, selectExpression, dataFilterClause); Skip skipIndex = skipIndex(attribute.getValueType(), attribute.hasOptionSet()); if (attribute.getValueType().isOrganisationUnit()) { @@ -234,7 +234,7 @@ protected List getColumnForAttribute(TrackedEntityAttribut .name(attribute.getUid()) .dimensionType(AnalyticsDimensionType.DYNAMIC) .dataType(dataType) - .selectExpression(sql) + .selectExpression(selectExpression) .skipIndex(skipIndex) .build()); From 55d8a84f9be5c206dac2d6a3ae128926f537b086 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 11:09:54 +0100 Subject: [PATCH 058/170] fix: Update code --- .../table/AbstractEventJdbcTableManager.java | 13 +++++++++++++ .../table/JdbcEventAnalyticsTableManager.java | 13 ------------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 8cf2e01dee72..22973bfe7c73 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -46,6 +46,7 @@ import org.hisp.dhis.analytics.table.model.Skip; import org.hisp.dhis.analytics.table.setting.AnalyticsTableSettings; import org.hisp.dhis.category.CategoryService; +import org.hisp.dhis.common.IdentifiableObject; import org.hisp.dhis.common.IdentifiableObjectManager; import org.hisp.dhis.common.ValueType; import org.hisp.dhis.commons.util.TextUtils; @@ -306,4 +307,16 @@ private String getSelectSubquery( "closingParentheses", getClosingParentheses(selectExpression), "attributeUid", quote(attribute.getUid()))); } + + /** + * Returns a map of identifiable properties and values. + * + * @param object the {@link IdentifiableObject}. + * @return a {@link Map}. + */ + protected Map toVariableMap(IdentifiableObject object) { + return Map.of( + "id", String.valueOf(object.getId()), + "uid", quote(object.getUid())); + } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index f7cf760b316b..4983383a6106 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -67,7 +67,6 @@ import org.hisp.dhis.calendar.Calendar; import org.hisp.dhis.category.Category; import org.hisp.dhis.category.CategoryService; -import org.hisp.dhis.common.IdentifiableObject; import org.hisp.dhis.common.IdentifiableObjectManager; import org.hisp.dhis.common.ValueType; import org.hisp.dhis.commons.collection.ListUtils; @@ -388,18 +387,6 @@ private String getAttributeValueJoinClause(Program program) { .collect(Collectors.joining()); } - /** - * Returns a variable map. - * - * @param object the object. - * @return a {@link Map}. - */ - private Map toVariableMap(IdentifiableObject object) { - return Map.of( - "id", String.valueOf(object.getId()), - "uid", quote(object.getUid())); - } - /** * Returns a partition SQL clause. * From 86ef201ef7a9c9dc0b285b4ef134327a1cba806d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 11:19:49 +0100 Subject: [PATCH 059/170] fix: Update code --- .../table/AbstractEventJdbcTableManager.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 22973bfe7c73..658f239a30ff 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -38,6 +38,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.partition.PartitionManager; import org.hisp.dhis.analytics.table.model.AnalyticsDimensionType; @@ -56,6 +57,7 @@ import org.hisp.dhis.db.sql.SqlBuilder; import org.hisp.dhis.organisationunit.OrganisationUnitService; import org.hisp.dhis.period.PeriodDataProvider; +import org.hisp.dhis.program.Program; import org.hisp.dhis.resourcetable.ResourceTableService; import org.hisp.dhis.setting.SystemSettingsProvider; import org.hisp.dhis.trackedentity.TrackedEntityAttribute; @@ -308,6 +310,24 @@ private String getSelectSubquery( "attributeUid", quote(attribute.getUid()))); } + /** + * Returns a join clause for attribute value for every attribute of the given program. + * + * @param program the {@link Program}. + * @return a join clause. + */ + protected String getAttributeValueJoinClause(Program program) { + String template = + """ + left join ${trackedentityattributevalue} as ${uid} \ + on en.trackedentityid=${uid}.trackedentityid \ + and ${uid}.trackedentityattributeid = ${id}\s"""; + + return program.getNonConfidentialTrackedEntityAttributes().stream() + .map(attribute -> replaceQualify(template, toVariableMap(attribute))) + .collect(Collectors.joining()); + } + /** * Returns a map of identifiable properties and values. * From f4e76843c7bb8b751b9b5d2963574e7bf4251b1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 11:20:29 +0100 Subject: [PATCH 060/170] fix: Update code --- .../table/JdbcEventAnalyticsTableManager.java | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 4983383a6106..f24e870b24d7 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -50,7 +50,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.Validate; import org.hisp.dhis.analytics.AnalyticsTableHookService; @@ -369,24 +368,6 @@ and ev.status in (${exportableEventStatues}) \ populateTableInternal(partition, fromClause); } - /** - * Returns a join clause for attribute value for every attribute of the given program. - * - * @param program the {@link Program}. - * @return a join clause. - */ - private String getAttributeValueJoinClause(Program program) { - String template = - """ - left join ${trackedentityattributevalue} as ${uid} \ - on en.trackedentityid=${uid}.trackedentityid \ - and ${uid}.trackedentityattributeid = ${id}\s"""; - - return program.getNonConfidentialTrackedEntityAttributes().stream() - .map(attribute -> replaceQualify(template, toVariableMap(attribute))) - .collect(Collectors.joining()); - } - /** * Returns a partition SQL clause. * From a0320721704126ef96026a886521f2095b662a07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 11:21:58 +0100 Subject: [PATCH 061/170] fix: Update code --- .../analytics/table/JdbcEnrollmentAnalyticsTableManager.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java index d18fbdffd80b..172745e793d3 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java @@ -136,6 +136,7 @@ protected List getPartitionChecks(Integer year, Date endDate) { @Override public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTablePartition partition) { Program program = partition.getMasterTable().getProgram(); + String attributeJoinClause = getAttributeValueJoinClause(program); String fromClause = replaceQualify( @@ -148,6 +149,7 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti left join analytics_rs_dateperiodstructure dps on cast(en.enrollmentdate as date)=dps.dateperiod \ left join analytics_rs_orgunitstructure ous on en.organisationunitid=ous.organisationunitid \ left join analytics_rs_organisationunitgroupsetstructure ougs on en.organisationunitid=ougs.organisationunitid \ + ${attributeJoinClause}\ where pr.programid=${programId} \ and en.organisationunitid is not null \ and (ougs.startdate is null or dps.monthstartdate=ougs.startdate) \ @@ -155,6 +157,7 @@ left join analytics_rs_dateperiodstructure dps on cast(en.enrollmentdate as date and en.occurreddate is not null \ and en.deleted = false\s""", Map.of( + "attributeJoinClause", attributeJoinClause, "programId", String.valueOf(program.getId()), "startTime", toLongDate(params.getStartTime()))); From 87a0fd313556d548556ad939c99e88343309c892 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 11:25:16 +0100 Subject: [PATCH 062/170] fix: Update code --- .../table/JdbcEventAnalyticsTableManagerTest.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java index b7da2447b4ac..fd650d2adabe 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java @@ -650,12 +650,7 @@ void verifyTeiTypeOrgUnitFetchesOuNameWhenPopulatingEventAnalyticsTable() { subject.populateTable(params, partition); verify(jdbcTemplate).execute(sql.capture()); - String ouUidQuery = - String.format( - """ - (select value from "trackedentityattributevalue" where trackedentityid=en.trackedentityid and \ - trackedentityattributeid=9999) as %s""", - quote(tea.getUid())); + String ouUidQuery = String.format("%s.value", quote(tea.getUid())); String ouNameQuery = String.format( From c93f4ff8d73766fffef4c191adbb1380529b9e0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 11:28:15 +0100 Subject: [PATCH 063/170] fix: Update code --- .../table/JdbcEventAnalyticsTableManagerTest.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java index fd650d2adabe..a1e886cd9001 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java @@ -537,6 +537,7 @@ void verifyGetTableWithTrackedEntityAttribute() { .withTableType(AnalyticsTableType.EVENT) .withColumnSize(59 + OU_NAME_HIERARCHY_COUNT) .addColumns(periodColumns) + // Org unit UID column .addColumn( d1.getUid(), TEXT, @@ -939,12 +940,7 @@ void verifyTeaTypeOrgUnitFetchesOuNameWhenPopulatingEventAnalyticsTable() { verify(jdbcTemplate).execute(sql.capture()); - String ouUidQuery = - String.format( - """ - select value from "trackedentityattributevalue" where trackedentityid=en.trackedentityid \ - and trackedentityattributeid=9999) as %s""", - quote(tea.getUid())); + String ouUidQuery = String.format("%s.value", quote(tea.getUid())); String ouNameQuery = String.format( From 8d9b40f044190ccc855dd0f7eb1ec444f067ac2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 11:35:09 +0100 Subject: [PATCH 064/170] fix: Update code --- .../table/JdbcEventAnalyticsTableManagerTest.java | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java index a1e886cd9001..b2da976c129e 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java @@ -501,10 +501,7 @@ void verifyGetTableWithTrackedEntityAttribute() { String aliasD1 = """ eventdatavalues #>> '{deabcdefghZ, value}' as "deabcdefghZ\""""; - String aliasTeaUid = - """ - (select value from "trackedentityattributevalue" where trackedentityid=en.trackedentityid \ - and trackedentityattributeid=%d) as "%s\""""; + String aliasTeaUid = "%s.value"; String aliasTea1 = """ @@ -537,13 +534,8 @@ void verifyGetTableWithTrackedEntityAttribute() { .withTableType(AnalyticsTableType.EVENT) .withColumnSize(59 + OU_NAME_HIERARCHY_COUNT) .addColumns(periodColumns) - // Org unit UID column - .addColumn( - d1.getUid(), - TEXT, - toSelectExpression(aliasD1, d1.getUid()), - Skip.SKIP) // ValueType.TEXT - .addColumn(tea1.getUid(), TEXT, String.format(aliasTeaUid, tea1.getId(), tea1.getUid())) + .addColumn(d1.getUid(), TEXT, toSelectExpression(aliasD1, d1.getUid()), Skip.SKIP) + .addColumn(tea1.getUid(), TEXT, String.format(aliasTeaUid, quote(tea1.getUid()))) // Org unit geometry column .addColumn( tea1.getUid() + "_geom", From 00536ff9d6bf09af4df48164a162aef69f46bf29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 11:36:03 +0100 Subject: [PATCH 065/170] fix: Update code --- .../table/JdbcEnrollmentAnalyticsTableManagerTest.java | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManagerTest.java index d1b9f4443599..f2f2d642cf42 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManagerTest.java @@ -137,13 +137,7 @@ void verifyTeiTypeOrgUnitFetchesOuUidWhenPopulatingEventAnalyticsTable() { subject.populateTable(params, partition); verify(jdbcTemplate).execute(sql.capture()); - String ouQuery = - format( - """ - (select value from "trackedentityattributevalue" \ - where trackedentityid=en.trackedentityid and \ - trackedentityattributeid=9999) as %s""", - quote(tea.getUid())); + String ouQuery = format("%s.value", quote(tea.getUid())); assertThat(sql.getValue(), containsString(ouQuery)); } From 15cb92a38989b7cfaff3322d6dbcc8aeb3ef0af2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 11:46:49 +0100 Subject: [PATCH 066/170] fix: Update code --- .../org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java index 966d49573cec..7f735ce1074b 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java @@ -427,7 +427,7 @@ protected AnalyticsTable getLatestAnalyticsTable( protected void invokeTimeAndLog(String sql, String logPattern, Object... args) { Timer timer = new SystemTimer().start(); - log.info("Populate table SQL: '{}'", sql); + log.debug("Populate table SQL: '{}'", sql); jdbcTemplate.execute(sql); From 85c2c92224ebf7606e54a53dc35ddefc03942b3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 11:50:44 +0100 Subject: [PATCH 067/170] fix: Update code --- .../table/AbstractEventJdbcTableManager.java | 6 +++--- .../table/JdbcEventAnalyticsTableManager.java | 19 ++++++++++--------- .../AbstractEventJdbcTableManagerTest.java | 10 +++++----- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 658f239a30ff..543c686379ac 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -120,13 +120,13 @@ protected Skip skipIndex(ValueType valueType, boolean hasOptionSet) { } /** - * Returns a select expression, potentially with a cast statement, based on the given value type. + * Returns a column expression, potentially with a cast statement, based on the given value type. * * @param valueType the {@link ValueType} to represent as database column type. * @param columnExpression the expression or name of the column to be selected. * @return a select expression appropriate for the given value type and context. */ - protected String getSelectExpression(ValueType valueType, String columnExpression) { + protected String getColumnExpression(ValueType valueType, String columnExpression) { if (valueType.isDecimal()) { return getCastExpression(columnExpression, NUMERIC_REGEXP, sqlBuilder.dataTypeDouble()); } else if (valueType.isInteger()) { @@ -224,7 +224,7 @@ protected List getColumnForAttribute(TrackedEntityAttribut String valueColumn = String.format("%s.%s", quote(attribute.getUid()), "value"); DataType dataType = getColumnType(attribute.getValueType(), isSpatialSupport()); - String selectExpression = getSelectExpression(attribute.getValueType(), valueColumn); + String selectExpression = getColumnExpression(attribute.getValueType(), valueColumn); String dataFilterClause = getDataFilterClause(attribute); Skip skipIndex = skipIndex(attribute.getValueType(), attribute.hasOptionSet()); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index f24e870b24d7..95a45283c8a6 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -489,15 +489,16 @@ private List getColumnForDataElement( List columns = new ArrayList<>(); DataType dataType = getColumnType(dataElement.getValueType(), isSpatialSupport()); - String columnExpression = + String jsonExpression = sqlBuilder.jsonExtractNested("eventdatavalues", dataElement.getUid(), "value"); - String selectExpression = getSelectExpression(dataElement.getValueType(), columnExpression); + String columnExpression = getColumnExpression(dataElement.getValueType(), jsonExpression); String dataFilterClause = getDataFilterClause(dataElement); - String sql = String.format("%s as %s", selectExpression, quote(dataElement.getUid())); + String selectExpression = + String.format("%s as %s", columnExpression, quote(dataElement.getUid())); Skip skipIndex = skipIndex(dataElement.getValueType(), dataElement.hasOptionSet()); if (withLegendSet) { - return getColumnFromDataElementWithLegendSet(dataElement, selectExpression, dataFilterClause); + return getColumnFromDataElementWithLegendSet(dataElement, columnExpression, dataFilterClause); } if (dataElement.getValueType().isOrganisationUnit()) { @@ -509,7 +510,7 @@ private List getColumnForDataElement( .name(dataElement.getUid()) .dimensionType(AnalyticsDimensionType.DYNAMIC) .dataType(dataType) - .selectExpression(sql) + .selectExpression(selectExpression) .skipIndex(skipIndex) .build()); @@ -590,7 +591,7 @@ private List getAttributeColumns(Program program) { */ private List getColumnForAttributeWithLegendSet( TrackedEntityAttribute attribute) { - String selectClause = getSelectExpression(attribute.getValueType(), "value"); + String columnExpression = getColumnExpression(attribute.getValueType(), "value"); String numericClause = getNumericClause(); String query = """ @@ -605,11 +606,11 @@ private List getColumnForAttributeWithLegendSet( .map( ls -> { String column = attribute.getUid() + PartitionUtils.SEP + ls.getUid(); - String sql = + String selectExpression = replaceQualify( query, Map.of( - "selectClause", selectClause, + "selectClause", columnExpression, "legendSetId", String.valueOf(ls.getId()), "column", column, "attributeId", String.valueOf(attribute.getId()), @@ -618,7 +619,7 @@ private List getColumnForAttributeWithLegendSet( return AnalyticsTableColumn.builder() .name(column) .dataType(CHARACTER_11) - .selectExpression(sql) + .selectExpression(selectExpression) .build(); }) .toList(); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManagerTest.java index 05f4337cc10a..ddb386b7bd0b 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManagerTest.java @@ -76,7 +76,7 @@ then cast(eventdatavalues #>> '{GieVkTxp4HH, value}' as double precision) \ else null end"""; String actual = - manager.getSelectExpression(ValueType.NUMBER, "eventdatavalues #>> '{GieVkTxp4HH, value}'"); + manager.getColumnExpression(ValueType.NUMBER, "eventdatavalues #>> '{GieVkTxp4HH, value}'"); assertEquals(expected, actual); } @@ -88,7 +88,7 @@ void testGetSelectExpressionBoolean() { case when eventdatavalues #>> '{Xl3voRRcmpo, value}' = 'true' then 1 when eventdatavalues #>> '{Xl3voRRcmpo, value}' = 'false' then 0 else null end"""; String actual = - manager.getSelectExpression( + manager.getColumnExpression( ValueType.BOOLEAN, "eventdatavalues #>> '{Xl3voRRcmpo, value}'"); assertEquals(expected, actual); @@ -103,7 +103,7 @@ then cast(eventdatavalues #>> '{AL04Wbutskk, value}' as timestamp) \ else null end"""; String actual = - manager.getSelectExpression(ValueType.DATE, "eventdatavalues #>> '{AL04Wbutskk, value}'"); + manager.getColumnExpression(ValueType.DATE, "eventdatavalues #>> '{AL04Wbutskk, value}'"); assertEquals(expected, actual); } @@ -115,7 +115,7 @@ void testGetSelectExpressionText() { eventdatavalues #>> '{FwUzmc49Pcr, value}'"""; String actual = - manager.getSelectExpression(ValueType.TEXT, "eventdatavalues #>> '{FwUzmc49Pcr, value}'"); + manager.getColumnExpression(ValueType.TEXT, "eventdatavalues #>> '{FwUzmc49Pcr, value}'"); assertEquals(expected, actual); } @@ -129,7 +129,7 @@ void testGetSelectExpressionGeometry() { ST_GeomFromGeoJSON('{"type":"Point", "coordinates":' || (eventdatavalues #>> '{C6bh7GevJfH, value}') || ', "crs":{"type":"name", "properties":{"name":"EPSG:4326"}}}')"""; String actual = - manager.getSelectExpression( + manager.getColumnExpression( ValueType.GEOJSON, "eventdatavalues #>> '{C6bh7GevJfH, value}'"); assertEquals(expected, actual); From bec3c7e8b19dcd2511ba4309d4c5edb93ca93658 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 13:33:31 +0100 Subject: [PATCH 068/170] fix: Update code --- .../table/AbstractEventJdbcTableManager.java | 45 ++++--------------- .../table/JdbcEventAnalyticsTableManager.java | 29 +++++++----- 2 files changed, 26 insertions(+), 48 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 543c686379ac..f8d9f1bd6f72 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -27,18 +27,17 @@ */ package org.hisp.dhis.analytics.table; -import static org.apache.commons.lang3.StringUtils.EMPTY; import static org.hisp.dhis.analytics.table.model.Skip.SKIP; import static org.hisp.dhis.analytics.util.AnalyticsUtils.getClosingParentheses; import static org.hisp.dhis.analytics.util.AnalyticsUtils.getColumnType; import static org.hisp.dhis.db.model.DataType.GEOMETRY; import static org.hisp.dhis.db.model.DataType.TEXT; -import static org.hisp.dhis.system.util.MathUtils.NUMERIC_LENIENT_REGEXP; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import org.apache.commons.lang3.Validate; import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.partition.PartitionManager; import org.hisp.dhis.analytics.table.model.AnalyticsDimensionType; @@ -99,14 +98,6 @@ public AbstractEventJdbcTableManager( public static final String OU_NAME_COL_SUFFIX = "_name"; - protected final String getNumericClause() { - return " and " + sqlBuilder.regexpMatch("value", "'" + NUMERIC_LENIENT_REGEXP + "'"); - } - - protected final String getDateClause() { - return " and " + sqlBuilder.regexpMatch("value", DATE_REGEXP); - } - /** * Indicates whether creating an index should be skipped. * @@ -148,22 +139,6 @@ protected String getColumnExpression(ValueType valueType, String columnExpressio } } - /** - * For numeric and date value types, returns a data filter clause for checking whether the value - * is valid according to the value type. For other value types, returns the empty string. - * - * @param attribute the {@link TrackedEntityAttribute}. - * @return a data filter clause. - */ - protected String getDataFilterClause(TrackedEntityAttribute attribute) { - if (attribute.isNumericType()) { - return getNumericClause(); - } else if (attribute.isDateType()) { - return getDateClause(); - } - return EMPTY; - } - /** * Returns a cast expression which includes a value filter for the given value type. * @@ -225,11 +200,10 @@ protected List getColumnForAttribute(TrackedEntityAttribut String valueColumn = String.format("%s.%s", quote(attribute.getUid()), "value"); DataType dataType = getColumnType(attribute.getValueType(), isSpatialSupport()); String selectExpression = getColumnExpression(attribute.getValueType(), valueColumn); - String dataFilterClause = getDataFilterClause(attribute); Skip skipIndex = skipIndex(attribute.getValueType(), attribute.hasOptionSet()); if (attribute.getValueType().isOrganisationUnit()) { - columns.addAll(getColumnForOrgUnitAttribute(attribute, dataFilterClause)); + columns.addAll(getColumnForOrgUnitAttribute(attribute)); } columns.add( @@ -248,11 +222,11 @@ protected List getColumnForAttribute(TrackedEntityAttribut * Returns a list of columns based on the given attribute. * * @param attribute the {@link TrackedEntityAttribute}. - * @param dataFilterClause the data filter clause. * @return a list of {@link AnalyticsTableColumn}. */ private List getColumnForOrgUnitAttribute( - TrackedEntityAttribute attribute, String dataFilterClause) { + TrackedEntityAttribute attribute) { + Validate.isTrue(attribute.getValueType().isOrganisationUnit()); List columns = new ArrayList<>(); String fromClause = @@ -260,7 +234,7 @@ private List getColumnForOrgUnitAttribute( if (isSpatialSupport()) { String selectExpression = "ou.geometry " + fromClause; - String ouGeoSql = getSelectSubquery(attribute, selectExpression, dataFilterClause); + String ouGeoSql = getSelectSubquery(attribute, selectExpression); columns.add( AnalyticsTableColumn.builder() .name((attribute.getUid() + OU_GEOMETRY_COL_SUFFIX)) @@ -272,7 +246,7 @@ private List getColumnForOrgUnitAttribute( } String selectExpression = "ou.name " + fromClause; - String ouNameSql = getSelectSubquery(attribute, selectExpression, dataFilterClause); + String ouNameSql = getSelectSubquery(attribute, selectExpression); columns.add( AnalyticsTableColumn.builder() @@ -291,20 +265,17 @@ private List getColumnForOrgUnitAttribute( * * @param attribute the {@link TrackedEntityAttribute}. * @param selectExpression the select expression. - * @param dataFilterClause the data filter clause. * @return a select statement. */ - private String getSelectSubquery( - TrackedEntityAttribute attribute, String selectExpression, String dataFilterClause) { + private String getSelectSubquery(TrackedEntityAttribute attribute, String selectExpression) { return replaceQualify( """ (select ${selectExpression} from ${trackedentityattributevalue} \ where trackedentityid=en.trackedentityid \ - and trackedentityattributeid=${attributeId}${dataFilterClause})\ + and trackedentityattributeid=${attributeId})\ ${closingParentheses} as ${attributeUid}""", Map.of( "selectExpression", selectExpression, - "dataFilterClause", dataFilterClause, "attributeId", String.valueOf(attribute.getId()), "closingParentheses", getClosingParentheses(selectExpression), "attributeUid", quote(attribute.getUid()))); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 95a45283c8a6..abdccfc70bb4 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -39,6 +39,7 @@ import static org.hisp.dhis.db.model.DataType.GEOMETRY; import static org.hisp.dhis.db.model.DataType.INTEGER; import static org.hisp.dhis.db.model.DataType.TEXT; +import static org.hisp.dhis.system.util.MathUtils.NUMERIC_LENIENT_REGEXP; import static org.hisp.dhis.util.DateUtils.toLongDate; import static org.hisp.dhis.util.DateUtils.toMediumDate; @@ -502,7 +503,7 @@ private List getColumnForDataElement( } if (dataElement.getValueType().isOrganisationUnit()) { - columns.addAll(getColumnForOrgUnitDataElement(dataElement, dataFilterClause)); + columns.addAll(getColumnForOrgUnitDataElement(dataElement)); } columns.add( @@ -524,8 +525,7 @@ private List getColumnForDataElement( * @param dataFilterClause the data filter SQL clause. * @return a list of {@link AnalyticsTableColumn}. */ - private List getColumnForOrgUnitDataElement( - DataElement dataElement, String dataFilterClause) { + private List getColumnForOrgUnitDataElement(DataElement dataElement) { List columns = new ArrayList<>(); String columnExpression = @@ -535,7 +535,7 @@ private List getColumnForOrgUnitDataElement( if (isSpatialSupport()) { String fromType = "ou.geometry " + fromClause; - String geoSql = getOrgUnitSelectExpression(dataElement, fromType, dataFilterClause); + String geoSql = getOrgUnitSelectExpression(dataElement, fromType); columns.add( AnalyticsTableColumn.builder() @@ -548,7 +548,7 @@ private List getColumnForOrgUnitDataElement( } String fromTypeSql = "ou.name " + fromClause; - String ouNameSql = getOrgUnitSelectExpression(dataElement, fromTypeSql, dataFilterClause); + String ouNameSql = getOrgUnitSelectExpression(dataElement, fromTypeSql); columns.add( AnalyticsTableColumn.builder() @@ -592,7 +592,7 @@ private List getAttributeColumns(Program program) { private List getColumnForAttributeWithLegendSet( TrackedEntityAttribute attribute) { String columnExpression = getColumnExpression(attribute.getValueType(), "value"); - String numericClause = getNumericClause(); + String numericClause = getNumericClause("value"); String query = """ \s(select l.uid from ${maplegend} l \ @@ -630,18 +630,15 @@ private List getColumnForAttributeWithLegendSet( * * @param dataElement the data element to create the select statement for. * @param selectExpression the select expression. - * @param dataFilterClause the data filter clause. * @return a select expression. */ - private String getOrgUnitSelectExpression( - DataElement dataElement, String selectExpression, String dataFilterClause) { + private String getOrgUnitSelectExpression(DataElement dataElement, String selectExpression) { Validate.isTrue(dataElement.getValueType().isOrganisationUnit()); String prts = getClosingParentheses(selectExpression); return replaceQualify( - "(select ${selectExpression} ${dataFilterClause})${closingParentheses} as ${uid}", + "(select ${selectExpression})${closingParentheses} as ${uid}", Map.of( "selectExpression", selectExpression, - "dataFilterClause", dataFilterClause, "closingParentheses", prts, "uid", quote(dataElement.getUid()))); } @@ -756,6 +753,16 @@ private List getDataYears( return jdbcTemplate.queryForList(sql, Integer.class); } + /** + * Returns a numeric regexp match expression for the given value. + * + * @param value the value. + * @return a numeric regexp match expression. + */ + private final String getNumericClause(String value) { + return " and " + sqlBuilder.regexpMatch(value, "'" + NUMERIC_LENIENT_REGEXP + "'"); + } + /** * Retrieve years for partition tables. Year will become a partition key. The default return value * is the list with the recent year. From f66b2beadb2dc30b6bb2ac1d8ef9dae29e316c64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 13:39:37 +0100 Subject: [PATCH 069/170] fix: Update code --- .../table/JdbcEventAnalyticsTableManagerDorisTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java index 9f952e8807b3..5b2bca0f3ec1 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java @@ -169,7 +169,7 @@ void verifyGetTableWithDataElements() { "case when json_unquote(json_extract(eventdatavalues, '$.%s.value')) regexp '^\\d{4}-\\d{2}-\\d{2}(\\s|T)?((\\d{2}:)(\\d{2}:)?(\\d{2}))?(|.(\\d{3})|.(\\d{3})Z)?$' then cast(json_unquote(json_extract(eventdatavalues, '$.%s.value')) as datetime) else null end as `%s`"; String aliasE = "json_unquote(json_extract(eventdatavalues, '$.%s.value')) as `%s`"; String aliasF = - "(select ou.name from dhis2.public.`organisationunit` ou where ou.uid = json_unquote(json_extract(eventdatavalues, '$.%s.value')) ) as `%s`"; + "(select ou.name from dhis2.public.`organisationunit` ou where ou.uid = json_unquote(json_extract(eventdatavalues, '$.%s.value'))) as `%s`"; String aliasG = "case when json_unquote(json_extract(eventdatavalues, '$.%s.value')) regexp '^(-?[0-9]+)(\\.[0-9]+)?$' then cast(json_unquote(json_extract(eventdatavalues, '$.%s.value')) as bigint) else null end as `%s`"; From 7b92571d30f15226c714a885c8f5dabe5bf76fad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 13:46:00 +0100 Subject: [PATCH 070/170] fix: Update code --- .../analytics/table/JdbcEventAnalyticsTableManagerTest.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java index b2da976c129e..f071d868106b 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java @@ -399,15 +399,13 @@ void verifyGetTableWithDataElements() { String aliasD5_geo = "(select ou.geometry from \"organisationunit\" ou where ou.uid = eventdatavalues #>> '{" + d5.getUid() - + ", value}' " - + ") as \"" + + ", value}') as \"" + d5.getUid() + "\""; String aliasD5_name = "(select ou.name from \"organisationunit\" ou where ou.uid = eventdatavalues #>> '{" + d5.getUid() - + ", value}' " - + ") as \"" + + ", value}') as \"" + d5.getUid() + "\""; AnalyticsTableUpdateParams params = From b400d6b309adfa34c62982ca3c724957cba46d46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 13:46:22 +0100 Subject: [PATCH 071/170] fix: Update code --- .../analytics/table/JdbcEventAnalyticsTableManagerTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java index f071d868106b..9a9db09b4f99 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java @@ -597,7 +597,7 @@ void verifyDataElementTypeOrgUnitFetchesOuNameWhenPopulatingEventAnalyticsTable( String.format( """ (select ou.name from "organisationunit" ou where ou.uid = \ - eventdatavalues #>> '{%s, value}' ) as %s""", + eventdatavalues #>> '{%s, value}') as %s""", d5.getUid(), quote(d5.getUid())); assertThat(sql.getValue(), containsString(ouUidQuery)); From f55b90005dc44235b7b8dc50b36a458a9014c4a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 13:47:24 +0100 Subject: [PATCH 072/170] fix: Update code --- .../analytics/table/JdbcEventAnalyticsTableManager.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index abdccfc70bb4..77eba78b4154 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -535,27 +535,27 @@ private List getColumnForOrgUnitDataElement(DataElement da if (isSpatialSupport()) { String fromType = "ou.geometry " + fromClause; - String geoSql = getOrgUnitSelectExpression(dataElement, fromType); + String ouGeoExpression = getOrgUnitSelectExpression(dataElement, fromType); columns.add( AnalyticsTableColumn.builder() .name((dataElement.getUid() + OU_GEOMETRY_COL_SUFFIX)) .dimensionType(AnalyticsDimensionType.DYNAMIC) .dataType(GEOMETRY) - .selectExpression(geoSql) + .selectExpression(ouGeoExpression) .indexType(IndexType.GIST) .build()); } String fromTypeSql = "ou.name " + fromClause; - String ouNameSql = getOrgUnitSelectExpression(dataElement, fromTypeSql); + String ouNameExpression = getOrgUnitSelectExpression(dataElement, fromTypeSql); columns.add( AnalyticsTableColumn.builder() .name((dataElement.getUid() + OU_NAME_COL_SUFFIX)) .dimensionType(AnalyticsDimensionType.DYNAMIC) .dataType(TEXT) - .selectExpression(ouNameSql) + .selectExpression(ouNameExpression) .skipIndex(SKIP) .build()); From fb8c59f8f98535674fc4c32ccb477229f3f75e11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 13:48:31 +0100 Subject: [PATCH 073/170] fix: Update code --- .../analytics/util/AnalyticsUtilsTest.java | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/util/AnalyticsUtilsTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/util/AnalyticsUtilsTest.java index 78424fe9c735..dbbb563aa2ac 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/util/AnalyticsUtilsTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/util/AnalyticsUtilsTest.java @@ -93,7 +93,6 @@ import org.hisp.dhis.test.TestBase; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; -import org.springframework.jdbc.BadSqlGrammarException; import org.springframework.jdbc.UncategorizedSQLException; /** @@ -763,31 +762,14 @@ void whenUncategorizedSQLException_withTableNotExisting_thenThrowException() { () -> AnalyticsUtils.withExceptionHandling(supplier, false)); } - @Test - void whenBadSqlGrammarException_withMultipleQueries_thenReturnEmpty() { - SQLException sqlException = new SQLException("syntax error"); - BadSqlGrammarException badSqlException = - new BadSqlGrammarException("task", DUMMY_SQL, sqlException); - Supplier supplier = - () -> { - throw badSqlException; - }; - - Optional result = AnalyticsUtils.withExceptionHandling(supplier, true); - - assertFalse(result.isPresent()); - } - @Test void whenQueryRuntimeException_thenRethrow() { - // Arrange QueryRuntimeException queryException = new QueryRuntimeException("Test error"); Supplier supplier = () -> { throw queryException; }; - // Act & Assert QueryRuntimeException thrown = assertThrows( QueryRuntimeException.class, From b4f34ba40f4fd220689370d36053d3f1f157534e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 13:50:36 +0100 Subject: [PATCH 074/170] fix: Update code --- .../analytics/table/JdbcEventAnalyticsTableManager.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 77eba78b4154..4474f2d4c97a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -535,27 +535,27 @@ private List getColumnForOrgUnitDataElement(DataElement da if (isSpatialSupport()) { String fromType = "ou.geometry " + fromClause; - String ouGeoExpression = getOrgUnitSelectExpression(dataElement, fromType); + String geoExpression = getOrgUnitSelectExpression(dataElement, fromType); columns.add( AnalyticsTableColumn.builder() .name((dataElement.getUid() + OU_GEOMETRY_COL_SUFFIX)) .dimensionType(AnalyticsDimensionType.DYNAMIC) .dataType(GEOMETRY) - .selectExpression(ouGeoExpression) + .selectExpression(geoExpression) .indexType(IndexType.GIST) .build()); } String fromTypeSql = "ou.name " + fromClause; - String ouNameExpression = getOrgUnitSelectExpression(dataElement, fromTypeSql); + String ouNameSql = getOrgUnitSelectExpression(dataElement, fromTypeSql); columns.add( AnalyticsTableColumn.builder() .name((dataElement.getUid() + OU_NAME_COL_SUFFIX)) .dimensionType(AnalyticsDimensionType.DYNAMIC) .dataType(TEXT) - .selectExpression(ouNameExpression) + .selectExpression(ouNameSql) .skipIndex(SKIP) .build()); From ddb11611195ad2c52e32877ea81e2ca40dfcdbcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 14:40:32 +0100 Subject: [PATCH 075/170] fix: Update code --- .../analytics/table/JdbcEventAnalyticsTableManager.java | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 4474f2d4c97a..02eff1c7ca53 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -30,7 +30,6 @@ import static java.util.stream.Collectors.toList; import static org.apache.commons.lang3.StringUtils.EMPTY; import static org.hisp.dhis.analytics.table.model.Skip.SKIP; -import static org.hisp.dhis.analytics.util.AnalyticsUtils.getClosingParentheses; import static org.hisp.dhis.analytics.util.AnalyticsUtils.getColumnType; import static org.hisp.dhis.commons.util.TextUtils.emptyIfTrue; import static org.hisp.dhis.commons.util.TextUtils.format; @@ -634,13 +633,9 @@ private List getColumnForAttributeWithLegendSet( */ private String getOrgUnitSelectExpression(DataElement dataElement, String selectExpression) { Validate.isTrue(dataElement.getValueType().isOrganisationUnit()); - String prts = getClosingParentheses(selectExpression); return replaceQualify( - "(select ${selectExpression})${closingParentheses} as ${uid}", - Map.of( - "selectExpression", selectExpression, - "closingParentheses", prts, - "uid", quote(dataElement.getUid()))); + "(select ${selectExpression}) as ${uid}", + Map.of("selectExpression", selectExpression, "uid", quote(dataElement.getUid()))); } /** From 251e246e4a77953b5941a4e5f9483cbaa2510cd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 15:09:32 +0100 Subject: [PATCH 076/170] fix: Update code --- .../table/JdbcEventAnalyticsTableManager.java | 34 ++++++++++++++----- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 02eff1c7ca53..34069bff8466 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -527,34 +527,52 @@ private List getColumnForDataElement( private List getColumnForOrgUnitDataElement(DataElement dataElement) { List columns = new ArrayList<>(); + String format = + """ + (select ou.${field} from ${organisationunit} ou \ + where ou.uid = ${columnExpression}) as ${uid}"""; String columnExpression = sqlBuilder.jsonExtractNested("eventdatavalues", dataElement.getUid(), "value"); - String fromClause = - qualifyVariables("from ${organisationunit} ou where ou.uid = " + columnExpression); if (isSpatialSupport()) { - String fromType = "ou.geometry " + fromClause; - String geoExpression = getOrgUnitSelectExpression(dataElement, fromType); + String geoSelectExpression = + replaceQualify( + format, + Map.of( + "field", + "geometry", + "columnExpression", + columnExpression, + "uid", + quote(dataElement.getUid()))); columns.add( AnalyticsTableColumn.builder() .name((dataElement.getUid() + OU_GEOMETRY_COL_SUFFIX)) .dimensionType(AnalyticsDimensionType.DYNAMIC) .dataType(GEOMETRY) - .selectExpression(geoExpression) + .selectExpression(geoSelectExpression) .indexType(IndexType.GIST) .build()); } - String fromTypeSql = "ou.name " + fromClause; - String ouNameSql = getOrgUnitSelectExpression(dataElement, fromTypeSql); + String nameSelectExpression = + replaceQualify( + format, + Map.of( + "field", + "name", + "columnExpression", + columnExpression, + "uid", + quote(dataElement.getUid()))); columns.add( AnalyticsTableColumn.builder() .name((dataElement.getUid() + OU_NAME_COL_SUFFIX)) .dimensionType(AnalyticsDimensionType.DYNAMIC) .dataType(TEXT) - .selectExpression(ouNameSql) + .selectExpression(nameSelectExpression) .skipIndex(SKIP) .build()); From d6c9f0e9af47d7c519d610a585a21dd7b253e7ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 15:09:45 +0100 Subject: [PATCH 077/170] fix: Update code --- .../table/JdbcEventAnalyticsTableManager.java | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 34069bff8466..35a31a623679 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -51,7 +51,6 @@ import java.util.Map; import java.util.Objects; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.Validate; import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.AnalyticsTableType; import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; @@ -642,20 +641,6 @@ private List getColumnForAttributeWithLegendSet( .toList(); } - /** - * Returns a select statement for the given select expression. - * - * @param dataElement the data element to create the select statement for. - * @param selectExpression the select expression. - * @return a select expression. - */ - private String getOrgUnitSelectExpression(DataElement dataElement, String selectExpression) { - Validate.isTrue(dataElement.getValueType().isOrganisationUnit()); - return replaceQualify( - "(select ${selectExpression}) as ${uid}", - Map.of("selectExpression", selectExpression, "uid", quote(dataElement.getUid()))); - } - /** * Returns a list of columns. * From 4bd41ca8c440b160a7f93ddbfc0d458125720fcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 15:11:34 +0100 Subject: [PATCH 078/170] fix: Update code --- .../table/JdbcEventAnalyticsTableManager.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 35a31a623679..9259dac18b64 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -532,18 +532,16 @@ private List getColumnForOrgUnitDataElement(DataElement da where ou.uid = ${columnExpression}) as ${uid}"""; String columnExpression = sqlBuilder.jsonExtractNested("eventdatavalues", dataElement.getUid(), "value"); + String alias = quote(dataElement.getUid()); if (isSpatialSupport()) { String geoSelectExpression = replaceQualify( format, Map.of( - "field", - "geometry", - "columnExpression", - columnExpression, - "uid", - quote(dataElement.getUid()))); + "field", "geometry", + "columnExpression", columnExpression, + "uid", alias)); columns.add( AnalyticsTableColumn.builder() From 427ffbba6103f8222cb688a8cb59055efbdc3f5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 15:13:16 +0100 Subject: [PATCH 079/170] fix: Update code --- .../analytics/table/JdbcEventAnalyticsTableManager.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 9259dac18b64..94f847d455e5 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -557,12 +557,9 @@ private List getColumnForOrgUnitDataElement(DataElement da replaceQualify( format, Map.of( - "field", - "name", - "columnExpression", - columnExpression, - "uid", - quote(dataElement.getUid()))); + "field", "name", + "columnExpression", columnExpression, + "uid", alias)); columns.add( AnalyticsTableColumn.builder() From b0f075191014f3e42820827a69027f10c6ad09d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 16:25:05 +0100 Subject: [PATCH 080/170] fix: Update code --- .../table/JdbcEventAnalyticsTableManager.java | 51 +++++++++---------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 94f847d455e5..ba3baefaf155 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -526,53 +526,52 @@ private List getColumnForDataElement( private List getColumnForOrgUnitDataElement(DataElement dataElement) { List columns = new ArrayList<>(); - String format = - """ - (select ou.${field} from ${organisationunit} ou \ - where ou.uid = ${columnExpression}) as ${uid}"""; - String columnExpression = - sqlBuilder.jsonExtractNested("eventdatavalues", dataElement.getUid(), "value"); - String alias = quote(dataElement.getUid()); - if (isSpatialSupport()) { - String geoSelectExpression = - replaceQualify( - format, - Map.of( - "field", "geometry", - "columnExpression", columnExpression, - "uid", alias)); - columns.add( AnalyticsTableColumn.builder() .name((dataElement.getUid() + OU_GEOMETRY_COL_SUFFIX)) .dimensionType(AnalyticsDimensionType.DYNAMIC) .dataType(GEOMETRY) - .selectExpression(geoSelectExpression) + .selectExpression(getOrgUnitSelectSubquery("geometry", dataElement)) .indexType(IndexType.GIST) .build()); } - String nameSelectExpression = - replaceQualify( - format, - Map.of( - "field", "name", - "columnExpression", columnExpression, - "uid", alias)); - columns.add( AnalyticsTableColumn.builder() .name((dataElement.getUid() + OU_NAME_COL_SUFFIX)) .dimensionType(AnalyticsDimensionType.DYNAMIC) .dataType(TEXT) - .selectExpression(nameSelectExpression) + .selectExpression(getOrgUnitSelectSubquery("name", dataElement)) .skipIndex(SKIP) .build()); return columns; } + /** + * Returns a org unit select query. + * + * @param column the column name. + * @param dataElement the {@link DataElement}. + * @return an org unit select query. + */ + private String getOrgUnitSelectSubquery(String column, DataElement dataElement) { + String format = + """ + (select ou.${column} from ${organisationunit} ou \ + where ou.uid = ${columnExpression}) as ${uid}"""; + String columnExpression = + sqlBuilder.jsonExtractNested("eventdatavalues", dataElement.getUid(), "value"); + String alias = quote(dataElement.getUid()); + return replaceQualify( + format, + Map.of( + "column", "name", + "columnExpression", columnExpression, + "uid", alias)); + } + /** * Returns columns for attributes of the given program. * From cce6d852f61d0aa0dda698f9fb622a6f4a29ae66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 16:30:46 +0100 Subject: [PATCH 081/170] fix: Update code --- .../dhis/analytics/table/JdbcEventAnalyticsTableManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index ba3baefaf155..65156116062b 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -567,7 +567,7 @@ private String getOrgUnitSelectSubquery(String column, DataElement dataElement) return replaceQualify( format, Map.of( - "column", "name", + "column", column, "columnExpression", columnExpression, "uid", alias)); } From 63484bd85b43918283e7509f18cf4ebbfae2a7e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 16:31:19 +0100 Subject: [PATCH 082/170] fix: Update code --- .../dhis/analytics/table/JdbcEventAnalyticsTableManager.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 65156116062b..94c8a2805487 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -560,16 +560,17 @@ private String getOrgUnitSelectSubquery(String column, DataElement dataElement) String format = """ (select ou.${column} from ${organisationunit} ou \ - where ou.uid = ${columnExpression}) as ${uid}"""; + where ou.uid = ${columnExpression}) as ${alias}"""; String columnExpression = sqlBuilder.jsonExtractNested("eventdatavalues", dataElement.getUid(), "value"); String alias = quote(dataElement.getUid()); + return replaceQualify( format, Map.of( "column", column, "columnExpression", columnExpression, - "uid", alias)); + "alias", alias)); } /** From 1064844f6aa9dfeeae36e1cae43564c0678cdb6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 20:30:02 +0100 Subject: [PATCH 083/170] fix: Update code --- .../table/AbstractEventJdbcTableManager.java | 1 - .../table/AbstractJdbcTableManager.java | 20 ++++++++++++++++--- ...AbstractJdbcEventAnalyticsManagerTest.java | 18 ++++++++++------- 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index f8d9f1bd6f72..f4f8034cc718 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -32,7 +32,6 @@ import static org.hisp.dhis.analytics.util.AnalyticsUtils.getColumnType; import static org.hisp.dhis.db.model.DataType.GEOMETRY; import static org.hisp.dhis.db.model.DataType.TEXT; - import java.util.ArrayList; import java.util.List; import java.util.Map; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java index 7f735ce1074b..449f4dafd43d 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java @@ -34,7 +34,6 @@ import static org.hisp.dhis.db.model.DataType.CHARACTER_11; import static org.hisp.dhis.db.model.DataType.TEXT; import static org.hisp.dhis.util.DateUtils.toLongDate; - import java.util.Collection; import java.util.Collections; import java.util.Date; @@ -42,9 +41,8 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; import org.hisp.dhis.analytics.AnalyticsTableHook; import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.AnalyticsTableManager; @@ -88,6 +86,8 @@ import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.util.Assert; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; /** * @author Lars Helge Overland @@ -672,6 +672,20 @@ protected boolean tableIsNotEmpty(String name) { return jdbcTemplate.queryForRowSet(sql).next(); } + /** + * Converts the given list of items to a comma-separated string, using the given mapping function to map the object to string. + * + * @param the type. + * @param list the list. + * @param mapper the mapping function. + * @return a comma-separated string. + */ + protected String toCommaSeparated(List list, Function mapper) { + return list.stream() + .map(mapper) + .collect(Collectors.joining(",")); + } + /** * Quotes the given relation. * diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManagerTest.java index cfa4d4c09173..ca2ae3937330 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManagerTest.java @@ -60,7 +60,6 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; - import java.math.BigDecimal; import java.math.RoundingMode; import java.sql.ResultSet; @@ -108,6 +107,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; +import org.mockito.Spy; import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.support.rowset.ResultSetWrappingSqlRowSet; @@ -125,9 +125,16 @@ class AbstractJdbcEventAnalyticsManagerTest extends EventAnalyticsTest { @Mock private ExecutionPlanStore executionPlanStore; @Mock private OrganisationUnitService organisationUnitService; + + @Spy private DefaultProgramIndicatorSubqueryBuilder programIndicatorSubqueryBuilder = + new DefaultProgramIndicatorSubqueryBuilder(programIndicatorService); + + @Spy private SqlBuilder sqlBuilder = new PostgreSqlBuilder(); - private final SqlBuilder sqlBuilder = new PostgreSqlBuilder(); + @Spy private EventTimeFieldSqlRenderer eventTimeFieldSqlRenderer = new EventTimeFieldSqlRenderer(sqlBuilder); + @Spy private EnrollmentTimeFieldSqlRenderer enrollmentTimeFieldSqlRenderer = new EnrollmentTimeFieldSqlRenderer(sqlBuilder); + private JdbcEventAnalyticsManager eventSubject; private JdbcEnrollmentAnalyticsManager enrollmentSubject; @@ -146,15 +153,12 @@ class AbstractJdbcEventAnalyticsManagerTest extends EventAnalyticsTest { @BeforeEach public void setUp() { - DefaultProgramIndicatorSubqueryBuilder programIndicatorSubqueryBuilder = - new DefaultProgramIndicatorSubqueryBuilder(programIndicatorService); - eventSubject = new JdbcEventAnalyticsManager( jdbcTemplate, programIndicatorService, programIndicatorSubqueryBuilder, - new EventTimeFieldSqlRenderer(sqlBuilder), + eventTimeFieldSqlRenderer, executionPlanStore, sqlBuilder); @@ -163,7 +167,7 @@ public void setUp() { jdbcTemplate, programIndicatorService, programIndicatorSubqueryBuilder, - new EnrollmentTimeFieldSqlRenderer(sqlBuilder), + enrollmentTimeFieldSqlRenderer, executionPlanStore, sqlBuilder); From 67ef7db28cc2b0abcd61b5defec739ddd893a46d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 20:31:02 +0100 Subject: [PATCH 084/170] fix: Update code --- .../table/AbstractEventJdbcTableManager.java | 1 + .../table/AbstractJdbcTableManager.java | 16 ++++---- ...AbstractJdbcEventAnalyticsManagerTest.java | 40 +++++++------------ 3 files changed, 23 insertions(+), 34 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index f4f8034cc718..f8d9f1bd6f72 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -32,6 +32,7 @@ import static org.hisp.dhis.analytics.util.AnalyticsUtils.getColumnType; import static org.hisp.dhis.db.model.DataType.GEOMETRY; import static org.hisp.dhis.db.model.DataType.TEXT; + import java.util.ArrayList; import java.util.List; import java.util.Map; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java index 449f4dafd43d..c7c63da03aae 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java @@ -34,6 +34,7 @@ import static org.hisp.dhis.db.model.DataType.CHARACTER_11; import static org.hisp.dhis.db.model.DataType.TEXT; import static org.hisp.dhis.util.DateUtils.toLongDate; + import java.util.Collection; import java.util.Collections; import java.util.Date; @@ -43,6 +44,8 @@ import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.hisp.dhis.analytics.AnalyticsTableHook; import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.AnalyticsTableManager; @@ -86,8 +89,6 @@ import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.util.Assert; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; /** * @author Lars Helge Overland @@ -673,19 +674,18 @@ protected boolean tableIsNotEmpty(String name) { } /** - * Converts the given list of items to a comma-separated string, using the given mapping function to map the object to string. - * + * Converts the given list of items to a comma-separated string, using the given mapping function + * to map the object to string. + * * @param the type. * @param list the list. * @param mapper the mapping function. * @return a comma-separated string. */ protected String toCommaSeparated(List list, Function mapper) { - return list.stream() - .map(mapper) - .collect(Collectors.joining(",")); + return list.stream().map(mapper).collect(Collectors.joining(",")); } - + /** * Quotes the given relation. * diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManagerTest.java index ca2ae3937330..6e108dbfb48f 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManagerTest.java @@ -106,6 +106,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Spy; import org.mockito.junit.jupiter.MockitoExtension; @@ -125,19 +126,24 @@ class AbstractJdbcEventAnalyticsManagerTest extends EventAnalyticsTest { @Mock private ExecutionPlanStore executionPlanStore; @Mock private OrganisationUnitService organisationUnitService; - - @Spy private DefaultProgramIndicatorSubqueryBuilder programIndicatorSubqueryBuilder = + + @Spy + private DefaultProgramIndicatorSubqueryBuilder programIndicatorSubqueryBuilder = new DefaultProgramIndicatorSubqueryBuilder(programIndicatorService); - + @Spy private SqlBuilder sqlBuilder = new PostgreSqlBuilder(); - @Spy private EventTimeFieldSqlRenderer eventTimeFieldSqlRenderer = new EventTimeFieldSqlRenderer(sqlBuilder); + @Spy + private EventTimeFieldSqlRenderer eventTimeFieldSqlRenderer = + new EventTimeFieldSqlRenderer(sqlBuilder); - @Spy private EnrollmentTimeFieldSqlRenderer enrollmentTimeFieldSqlRenderer = new EnrollmentTimeFieldSqlRenderer(sqlBuilder); - - private JdbcEventAnalyticsManager eventSubject; + @Spy + private EnrollmentTimeFieldSqlRenderer enrollmentTimeFieldSqlRenderer = + new EnrollmentTimeFieldSqlRenderer(sqlBuilder); - private JdbcEnrollmentAnalyticsManager enrollmentSubject; + @InjectMocks private JdbcEventAnalyticsManager eventSubject; + + @InjectMocks private JdbcEnrollmentAnalyticsManager enrollmentSubject; private Program programA; @@ -153,24 +159,6 @@ class AbstractJdbcEventAnalyticsManagerTest extends EventAnalyticsTest { @BeforeEach public void setUp() { - eventSubject = - new JdbcEventAnalyticsManager( - jdbcTemplate, - programIndicatorService, - programIndicatorSubqueryBuilder, - eventTimeFieldSqlRenderer, - executionPlanStore, - sqlBuilder); - - enrollmentSubject = - new JdbcEnrollmentAnalyticsManager( - jdbcTemplate, - programIndicatorService, - programIndicatorSubqueryBuilder, - enrollmentTimeFieldSqlRenderer, - executionPlanStore, - sqlBuilder); - programA = createProgram('A'); dataElementA = createDataElement('A', ValueType.INTEGER, AggregationType.SUM); From a4084c5ae9799a4974cd373586aa8541cc7bdeb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 20:31:05 +0100 Subject: [PATCH 085/170] fix: Update code --- .../event/data/AbstractJdbcEventAnalyticsManagerTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManagerTest.java index 6e108dbfb48f..bbade801c5f3 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManagerTest.java @@ -60,6 +60,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; + import java.math.BigDecimal; import java.math.RoundingMode; import java.sql.ResultSet; From 1ed29a486d3b9064287fdc5d5f48978653a76628 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 20:32:44 +0100 Subject: [PATCH 086/170] fix: Update code --- .../event/data/AbstractJdbcEventAnalyticsManagerTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManagerTest.java index bbade801c5f3..538f509a6f67 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManagerTest.java @@ -75,6 +75,7 @@ import org.hisp.dhis.analytics.AnalyticsAggregationType; import org.hisp.dhis.analytics.EventOutputType; import org.hisp.dhis.analytics.analyze.ExecutionPlanStore; +import org.hisp.dhis.analytics.common.ProgramIndicatorSubqueryBuilder; import org.hisp.dhis.analytics.event.EventQueryParams; import org.hisp.dhis.analytics.event.EventQueryParams.Builder; import org.hisp.dhis.analytics.event.data.programindicator.DefaultProgramIndicatorSubqueryBuilder; @@ -129,7 +130,7 @@ class AbstractJdbcEventAnalyticsManagerTest extends EventAnalyticsTest { @Mock private OrganisationUnitService organisationUnitService; @Spy - private DefaultProgramIndicatorSubqueryBuilder programIndicatorSubqueryBuilder = + private ProgramIndicatorSubqueryBuilder programIndicatorSubqueryBuilder = new DefaultProgramIndicatorSubqueryBuilder(programIndicatorService); @Spy private SqlBuilder sqlBuilder = new PostgreSqlBuilder(); @@ -161,7 +162,6 @@ class AbstractJdbcEventAnalyticsManagerTest extends EventAnalyticsTest { @BeforeEach public void setUp() { programA = createProgram('A'); - dataElementA = createDataElement('A', ValueType.INTEGER, AggregationType.SUM); dataElementA.setUid("fWIAEtYVEGk"); } From ee94d6521deb1432348a81e4e84e763067eda20f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 20:33:08 +0100 Subject: [PATCH 087/170] fix: Update code --- .../event/data/AbstractJdbcEventAnalyticsManagerTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManagerTest.java index 538f509a6f67..b37709db626e 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManagerTest.java @@ -234,8 +234,6 @@ void verifyGetCoordinateColumn() { + colName + ")::numeric, 6) || ']' as " + colName)); - - return; } @Test From 7918f3fcabb07f1bc057f4823ab516e84ec3cbee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 20:39:39 +0100 Subject: [PATCH 088/170] fix: Update code --- .../table/AbstractEventJdbcTableManager.java | 15 ++++----------- .../AbstractEventJdbcTableManagerTest.java | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index f8d9f1bd6f72..9392bfc49806 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -49,7 +49,6 @@ import org.hisp.dhis.common.IdentifiableObject; import org.hisp.dhis.common.IdentifiableObjectManager; import org.hisp.dhis.common.ValueType; -import org.hisp.dhis.commons.util.TextUtils; import org.hisp.dhis.dataapproval.DataApprovalLevelService; import org.hisp.dhis.db.model.DataType; import org.hisp.dhis.db.model.IndexType; @@ -171,19 +170,13 @@ protected void populateTableInternal(AnalyticsTablePartition partition, String f String sql = "insert into " + tableName + " ("; - for (AnalyticsTableColumn col : columns) { - sql += quote(col.getName()) + ","; - } - - sql = TextUtils.removeLastComma(sql) + ") select "; + sql += toCommaSeparated(columns, col -> quote(col.getName())); - for (AnalyticsTableColumn col : columns) { - sql += col.getSelectExpression() + ","; - } + sql += ") select "; - sql = TextUtils.removeLastComma(sql) + " "; + sql += toCommaSeparated(columns, AnalyticsTableColumn::getSelectExpression); - sql += fromClause; + sql += " " + fromClause; invokeTimeAndLog(sql, "Populating table: '{}'", tableName); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManagerTest.java index ddb386b7bd0b..a4ea5e5fa65f 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManagerTest.java @@ -30,8 +30,11 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.when; +import java.util.List; import org.hisp.dhis.analytics.table.setting.AnalyticsTableSettings; import org.hisp.dhis.common.ValueType; +import org.hisp.dhis.db.model.Column; +import org.hisp.dhis.db.model.DataType; import org.hisp.dhis.db.sql.PostgreSqlBuilder; import org.hisp.dhis.db.sql.SqlBuilder; import org.junit.jupiter.api.Test; @@ -50,6 +53,21 @@ class AbstractEventJdbcTableManagerTest { @InjectMocks private JdbcEventAnalyticsTableManager manager; + @Test + void testToCommaSeparated() { + List columns = + List.of( + new Column("dx", DataType.VARCHAR_255), + new Column("pe", DataType.VARCHAR_255), + new Column("value", DataType.DOUBLE)); + + String expected = + """ + "dx","pe","value\""""; + + assertEquals(expected, manager.toCommaSeparated(columns, col -> manager.quote(col.getName()))); + } + @Test void testGetCastExpression() { String expected = From 000cc5d01b522961d1ae90dff27c2a52e08ed1bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 20:47:49 +0100 Subject: [PATCH 089/170] fix: Update code --- ...ollmentAnalyticsDimensionsServiceTest.java | 23 ++++++++----------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EnrollmentAnalyticsDimensionsServiceTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EnrollmentAnalyticsDimensionsServiceTest.java index 66e7eb8dec26..ea1f01b27015 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EnrollmentAnalyticsDimensionsServiceTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EnrollmentAnalyticsDimensionsServiceTest.java @@ -27,8 +27,6 @@ */ package org.hisp.dhis.analytics.event.data; -import static org.hisp.dhis.analytics.common.AnalyticsDimensionsTestSupport.allValueTypeDataElements; -import static org.hisp.dhis.analytics.common.AnalyticsDimensionsTestSupport.allValueTypeTEAs; import static org.hisp.dhis.analytics.common.DimensionServiceCommonTest.aggregateAllowedValueTypesPredicate; import static org.hisp.dhis.analytics.common.DimensionServiceCommonTest.queryDisallowedValueTypesPredicate; import static org.hisp.dhis.test.TestBase.injectSecurityContextNoSettings; @@ -37,38 +35,35 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import java.util.Collections; import java.util.List; -import org.hisp.dhis.analytics.event.EnrollmentAnalyticsDimensionsService; import org.hisp.dhis.common.BaseIdentifiableObject; import org.hisp.dhis.common.PrefixedDimension; import org.hisp.dhis.dataelement.DataElement; import org.hisp.dhis.program.Program; import org.hisp.dhis.program.ProgramService; -import org.hisp.dhis.security.acl.AclService; import org.hisp.dhis.trackedentity.TrackedEntityAttribute; import org.hisp.dhis.user.SystemUser; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +@ExtendWith(MockitoExtension.class) class EnrollmentAnalyticsDimensionsServiceTest { - private EnrollmentAnalyticsDimensionsService enrollmentAnalyticsDimensionsService; + @Mock private ProgramService programService; + + @InjectMocks + private DefaultEnrollmentAnalyticsDimensionsService enrollmentAnalyticsDimensionsService; @BeforeEach void setup() { injectSecurityContextNoSettings(new SystemUser()); - ProgramService programService = mock(ProgramService.class); - Program program = mock(Program.class); when(programService.getProgram(any())).thenReturn(program); - when(program.getDataElements()).thenReturn(allValueTypeDataElements()); - when(program.getProgramIndicators()).thenReturn(Collections.emptySet()); - when(program.getTrackedEntityAttributes()).thenReturn(allValueTypeTEAs()); - - enrollmentAnalyticsDimensionsService = - new DefaultEnrollmentAnalyticsDimensionsService(programService, mock(AclService.class)); } @Test From 8caf620e6ba995f4d0cdbffc1f877d5911451f66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 20:52:10 +0100 Subject: [PATCH 090/170] fix: Update code --- .../event/data/EnrollmentAnalyticsManagerTest.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EnrollmentAnalyticsManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EnrollmentAnalyticsManagerTest.java index a389425ecada..6e1491910583 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EnrollmentAnalyticsManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EnrollmentAnalyticsManagerTest.java @@ -85,6 +85,7 @@ import org.mockito.ArgumentCaptor; import org.mockito.Captor; import org.mockito.Mock; +import org.mockito.Spy; import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; @@ -108,7 +109,11 @@ class EnrollmentAnalyticsManagerTest extends EventAnalyticsTest { @Mock private ProgramIndicatorService programIndicatorService; - private final SqlBuilder sqlBuilder = new PostgreSqlBuilder(); + @Spy private SqlBuilder sqlBuilder = new PostgreSqlBuilder(); + + @Spy + private EnrollmentTimeFieldSqlRenderer enrollmentTimeFieldSqlRenderer = + new EnrollmentTimeFieldSqlRenderer(sqlBuilder); @Captor private ArgumentCaptor sql; @@ -133,7 +138,7 @@ public void setUp() { jdbcTemplate, programIndicatorService, programIndicatorSubqueryBuilder, - new EnrollmentTimeFieldSqlRenderer(sqlBuilder), + enrollmentTimeFieldSqlRenderer, executionPlanStore, sqlBuilder); } From aeca0bd067e6cfbe9e243744d813ac7613c4c919 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 20:53:02 +0100 Subject: [PATCH 091/170] fix: Update code --- .../event/data/EnrollmentQueryHelperTest.java | 23 +------------------ 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EnrollmentQueryHelperTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EnrollmentQueryHelperTest.java index 2b11e9b0cc5c..267fd2d19c94 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EnrollmentQueryHelperTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EnrollmentQueryHelperTest.java @@ -35,7 +35,7 @@ import static org.hisp.dhis.common.OrganisationUnitSelectionMode.CHILDREN; import static org.hisp.dhis.common.OrganisationUnitSelectionMode.SELECTED; import static org.hisp.dhis.period.RelativePeriodEnum.LAST_3_DAYS; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.util.List; import java.util.Set; @@ -52,7 +52,6 @@ class EnrollmentQueryHelperTest { @Test void testGetHeaderColumnsSamePrefixedDimension() { - // Given List headers = List.of( new GridHeader("name0"), @@ -64,10 +63,8 @@ void testGetHeaderColumnsSamePrefixedDimension() { String sql = "select name0, name1, dim.name2, pe, value, ou from table"; - // When Set headerColumns = EnrollmentQueryHelper.getHeaderColumns(headers, sql); - // Then String[] columns = headerColumns.toArray(String[]::new); assertEquals(3, columns.length); assertEquals("t1.\"name0\"", columns[0]); @@ -77,7 +74,6 @@ void testGetHeaderColumnsSamePrefixedDimension() { @Test void testGetHeaderColumnsDifferentPrefixedDimension() { - // Given List headers = List.of( new GridHeader("name0"), @@ -89,10 +85,8 @@ void testGetHeaderColumnsDifferentPrefixedDimension() { String sql = "select name0, name1, ax.name2, pe, value, ou from table"; - // When Set headerColumns = EnrollmentQueryHelper.getHeaderColumns(headers, sql); - // Then String[] columns = headerColumns.toArray(String[]::new); assertEquals(3, columns.length); assertEquals("t1.\"name0\"", columns[0]); @@ -102,7 +96,6 @@ void testGetHeaderColumnsDifferentPrefixedDimension() { @Test void testGetOrgUnitLevelColumnsOuMode() { - // Given OrganisationUnit organisationUnit = new OrganisationUnit("OrgTest"); organisationUnit.setPath("/Level1/OrgTest"); @@ -114,10 +107,8 @@ void testGetOrgUnitLevelColumnsOuMode() { ORGUNIT_DIM_ID, ORGANISATION_UNIT, List.of(organisationUnit))) .build(); - // When Set orgUnitColumns = EnrollmentQueryHelper.getOrgUnitLevelColumns(params); - // Then String[] columns = orgUnitColumns.toArray(String[]::new); assertEquals(1, columns.length); assertEquals("uidlevel2", columns[0]); @@ -125,7 +116,6 @@ void testGetOrgUnitLevelColumnsOuMode() { @Test void testGetOrgUnitLevelColumnsOuModeSelected() { - // Given OrganisationUnit organisationUnit = new OrganisationUnit("/Level1/Level2"); EventQueryParams params = @@ -136,16 +126,13 @@ void testGetOrgUnitLevelColumnsOuModeSelected() { ORGUNIT_DIM_ID, ORGANISATION_UNIT, List.of(organisationUnit))) .build(); - // When Set orgUnitColumns = EnrollmentQueryHelper.getOrgUnitLevelColumns(params); - // Then assertEquals(0, orgUnitColumns.size()); } @Test void testGetOrgUnitLevelColumnsOuModeChildren() { - // Given OrganisationUnit organisationUnit = new OrganisationUnit("/Level1/Level2"); EventQueryParams params = @@ -156,16 +143,13 @@ void testGetOrgUnitLevelColumnsOuModeChildren() { ORGUNIT_DIM_ID, ORGANISATION_UNIT, List.of(organisationUnit))) .build(); - // When Set orgUnitColumns = EnrollmentQueryHelper.getOrgUnitLevelColumns(params); - // Then assertEquals(0, orgUnitColumns.size()); } @Test void testGetPeriodColumns() { - // Given Period period = new Period(LAST_3_DAYS); period.setPeriodType(PeriodType.getPeriodTypeFromIsoString("201101")); @@ -174,10 +158,8 @@ void testGetPeriodColumns() { .addDimension(new BaseDimensionalObject(PERIOD_DIM_ID, PERIOD, List.of(period))) .build(); - // When Set periodColumns = EnrollmentQueryHelper.getPeriodColumns(params); - // Then String[] columns = periodColumns.toArray(String[]::new); assertEquals(1, columns.length); assertEquals("t1.Monthly", columns[0]); @@ -185,16 +167,13 @@ void testGetPeriodColumns() { @Test void testGetPeriodColumnsNoPeriods() { - // Given EventQueryParams params = new EventQueryParams.Builder() .addDimension(new BaseDimensionalObject(ORGUNIT_DIM_ID, ORGANISATION_UNIT, List.of())) .build(); - // When Set periodColumns = EnrollmentQueryHelper.getPeriodColumns(params); - // Then assertEquals(0, periodColumns.size()); } } From 75f1e638b8882a803782387fe11d805a458ea065 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Thu, 5 Dec 2024 11:25:16 +0100 Subject: [PATCH 092/170] fix: Update code --- .../table/AbstractEventJdbcTableManager.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 9392bfc49806..658627459339 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -190,7 +190,7 @@ protected void populateTableInternal(AnalyticsTablePartition partition, String f protected List getColumnForAttribute(TrackedEntityAttribute attribute) { List columns = new ArrayList<>(); - String valueColumn = String.format("%s.%s", quote(attribute.getUid()), "value"); + String valueColumn = getValueColumn(attribute); DataType dataType = getColumnType(attribute.getValueType(), isSpatialSupport()); String selectExpression = getColumnExpression(attribute.getValueType(), valueColumn); Skip skipIndex = skipIndex(attribute.getValueType(), attribute.hasOptionSet()); @@ -253,6 +253,16 @@ private List getColumnForOrgUnitAttribute( return columns; } + /** + * Returns the value column with alias. + * + * @param attribute the {@link TrackedEntityAttribute}. + * @return the vlaue column with alias. + */ + private String getValueColumn(TrackedEntityAttribute attribute) { + return String.format("%s.%s", quote(attribute.getUid()), "value"); + } + /** * The select subquery statement. * From c3597a6f8fda2b5c14efed6794789cac517ac0e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Fri, 6 Dec 2024 20:05:55 +0100 Subject: [PATCH 093/170] fix: Update strings --- .../org/hisp/dhis/system/grid/ListGrid.java | 60 ++++++++----------- 1 file changed, 25 insertions(+), 35 deletions(-) diff --git a/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/grid/ListGrid.java b/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/grid/ListGrid.java index d7266b8c5368..17d4eb7c8e89 100644 --- a/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/grid/ListGrid.java +++ b/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/grid/ListGrid.java @@ -31,11 +31,6 @@ import static org.hisp.dhis.common.ValueType.getValueTypeFromSqlType; import static org.hisp.dhis.common.collection.CollectionUtils.mapToList; import static org.hisp.dhis.feedback.ErrorCode.E7230; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import com.google.common.collect.Iterables; import java.io.Serializable; import java.sql.ResultSet; import java.sql.ResultSetMetaData; @@ -52,10 +47,6 @@ import java.util.Set; import java.util.TreeMap; import java.util.regex.Pattern; -import lombok.AccessLevel; -import lombok.Getter; -import net.sf.jasperreports.engine.JRException; -import net.sf.jasperreports.engine.JRField; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.Validate; import org.apache.commons.math3.stat.regression.SimpleRegression; @@ -72,6 +63,14 @@ import org.hisp.dhis.system.util.MathUtils; import org.springframework.jdbc.support.rowset.SqlRowSet; import org.springframework.jdbc.support.rowset.SqlRowSetMetaData; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.google.common.collect.Iterables; +import lombok.AccessLevel; +import lombok.Getter; +import net.sf.jasperreports.engine.JRException; +import net.sf.jasperreports.engine.JRField; /** * @author Lars Helge Overland @@ -502,12 +501,9 @@ public Grid addColumn(List columnValues) { int currentColumnIndex = 0; if (grid.size() != columnValues.size()) { - throw new IllegalStateException( - "Number of column values (" - + columnValues.size() - + ") is not equal to number of rows (" - + grid.size() - + ")"); + throw new IllegalStateException(String.format( + "Number of column values (%d) is not equal to number of rows (%d)", + columnValues.size(), grid.size())); } for (int i = 0; i < grid.size(); i++) { @@ -525,12 +521,9 @@ public Grid addColumn(int columnIndex, List columnValues) { int currentColumnIndex = 0; if (grid.size() != columnValues.size()) { - throw new IllegalStateException( - "Number of column values (" - + columnValues.size() - + ") is not equal to number of rows (" - + grid.size() - + ")"); + throw new IllegalStateException(String.format( + "Number of column values (%d) is not equal to number of rows (%d)", + columnValues.size(), grid.size())); } for (int i = 0; i < grid.size(); i++) { @@ -657,7 +650,7 @@ public Grid limitGrid(int limit) { public Grid limitGrid(int startPos, int endPos) { if (startPos < 0 || endPos < startPos || endPos > getHeight()) { throw new IllegalStateException( - "Illegal start / end pos: " + startPos + ", " + endPos + ", " + getHeight()); + "Illegal start or end pos: " + startPos + ", " + endPos + ", " + getHeight()); } grid = grid.subList(startPos, endPos); @@ -806,16 +799,15 @@ public Grid substituteMetaData(Map metaDataMap) { GridHeader header = headers.get(colIndex); // Header - + Object headerMetaName = metaDataMap.get(header.getName()); if (headerMetaName != null) { header.setName(String.valueOf(headerMetaName)); } + // Column cells if (header.isMeta()) { - // Column cells - substituteMetaData(colIndex, colIndex, metaDataMap); } } @@ -1101,7 +1093,8 @@ public void repositionColumns(List columnIndexes) { row.addAll(orderedValues); } - // reposition columns in the row context structure + // Reposition columns in the row context structure + Map> orderedRowContext = new HashMap<>(); for (Map.Entry> rowContextEntry : rowContext.entrySet()) { @@ -1113,7 +1106,8 @@ public void repositionColumns(List columnIndexes) { .forEach( key -> { if (numberRegex.matcher(key).matches()) { - // reindexing of columns + // Reindexing of columns + orderedRowContextItems.put( columnIndexes.get(Integer.parseInt(key)).toString(), ctxItem.get(key)); } @@ -1150,13 +1144,9 @@ private void verifyGridState() { for (List row : grid) { if (rowLength != null && rowLength != row.size()) { - throw new IllegalStateException( - "Grid rows do not have the same number of cells, previous: " - + rowLength - + ", this: " - + row.size() - + ", at row: " - + rowPos); + throw new IllegalStateException(String.format( + "Grid rows do not have the same number of cells, previous: %d, this: %d, at row: %d", + rowLength, row.size(), rowPos)); } rowPos++; @@ -1177,7 +1167,7 @@ private void updateColumnIndexMap() { } // ------------------------------------------------------------------------- - // toString + // ToString // ------------------------------------------------------------------------- @Override From bec56ad0d0c7e878435439ca1c864dbae616d59f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 10 Dec 2024 17:59:49 +0100 Subject: [PATCH 094/170] fix: Update code --- .../table/AbstractEventJdbcTableManager.java | 46 ++++++++----------- .../JdbcEnrollmentAnalyticsTableManager.java | 3 +- .../table/JdbcEventAnalyticsTableManager.java | 12 +++-- 3 files changed, 29 insertions(+), 32 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 658627459339..c6eb428842c1 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -28,11 +28,9 @@ package org.hisp.dhis.analytics.table; import static org.hisp.dhis.analytics.table.model.Skip.SKIP; -import static org.hisp.dhis.analytics.util.AnalyticsUtils.getClosingParentheses; import static org.hisp.dhis.analytics.util.AnalyticsUtils.getColumnType; import static org.hisp.dhis.db.model.DataType.GEOMETRY; import static org.hisp.dhis.db.model.DataType.TEXT; - import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -50,6 +48,7 @@ import org.hisp.dhis.common.IdentifiableObjectManager; import org.hisp.dhis.common.ValueType; import org.hisp.dhis.dataapproval.DataApprovalLevelService; +import org.hisp.dhis.dataelement.DataElement; import org.hisp.dhis.db.model.DataType; import org.hisp.dhis.db.model.IndexType; import org.hisp.dhis.db.sql.SqlBuilder; @@ -222,31 +221,23 @@ private List getColumnForOrgUnitAttribute( Validate.isTrue(attribute.getValueType().isOrganisationUnit()); List columns = new ArrayList<>(); - String fromClause = - qualifyVariables("from ${organisationunit} ou where ou.uid = (select value"); - if (isSpatialSupport()) { - String selectExpression = "ou.geometry " + fromClause; - String ouGeoSql = getSelectSubquery(attribute, selectExpression); columns.add( AnalyticsTableColumn.builder() .name((attribute.getUid() + OU_GEOMETRY_COL_SUFFIX)) .dimensionType(AnalyticsDimensionType.DYNAMIC) .dataType(GEOMETRY) - .selectExpression(ouGeoSql) + .selectExpression(getOrgUnitSelectSubquery(attribute, "geometry")) .indexType(IndexType.GIST) .build()); } - String selectExpression = "ou.name " + fromClause; - String ouNameSql = getSelectSubquery(attribute, selectExpression); - columns.add( AnalyticsTableColumn.builder() .name((attribute.getUid() + OU_NAME_COL_SUFFIX)) .dimensionType(AnalyticsDimensionType.DYNAMIC) .dataType(TEXT) - .selectExpression(ouNameSql) + .selectExpression(getOrgUnitSelectSubquery(attribute, "name")) .skipIndex(SKIP) .build()); @@ -264,24 +255,27 @@ private String getValueColumn(TrackedEntityAttribute attribute) { } /** - * The select subquery statement. + * Returns a org unit select query. * - * @param attribute the {@link TrackedEntityAttribute}. - * @param selectExpression the select expression. - * @return a select statement. + * @param column the column name. + * @param dataElement the {@link DataElement}. + * @return an org unit select query. */ - private String getSelectSubquery(TrackedEntityAttribute attribute, String selectExpression) { - return replaceQualify( + private String getOrgUnitSelectSubquery(TrackedEntityAttribute attribute, String column) { + String format = """ - (select ${selectExpression} from ${trackedentityattributevalue} \ - where trackedentityid=en.trackedentityid \ - and trackedentityattributeid=${attributeId})\ - ${closingParentheses} as ${attributeUid}""", + (select ou.${column} from ${organisationunit} ou \ + where ou.uid = ${columnExpression}) as ${alias}"""; + String valueColumn = getValueColumn(attribute); + String columnExpression = getColumnExpression(attribute.getValueType(), valueColumn); + String alias = quote(attribute.getUid()); + + return replaceQualify( + format, Map.of( - "selectExpression", selectExpression, - "attributeId", String.valueOf(attribute.getId()), - "closingParentheses", getClosingParentheses(selectExpression), - "attributeUid", quote(attribute.getUid()))); + "column", column, + "columnExpression", columnExpression, + "alias", alias)); } /** diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java index 172745e793d3..5ac39cc8f5b7 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java @@ -34,6 +34,7 @@ import java.util.Date; import java.util.List; import java.util.Map; + import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.AnalyticsTableType; import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; @@ -150,7 +151,7 @@ left join analytics_rs_dateperiodstructure dps on cast(en.enrollmentdate as date left join analytics_rs_orgunitstructure ous on en.organisationunitid=ous.organisationunitid \ left join analytics_rs_organisationunitgroupsetstructure ougs on en.organisationunitid=ougs.organisationunitid \ ${attributeJoinClause}\ - where pr.programid=${programId} \ + where pr.programid = ${programId} \ and en.organisationunitid is not null \ and (ougs.startdate is null or dps.monthstartdate=ougs.startdate) \ and en.lastupdated <= '${startTime}' \ diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 94c8a2805487..b545b7d893ed 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -50,7 +50,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import lombok.extern.slf4j.Slf4j; + import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.AnalyticsTableType; import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; @@ -89,6 +89,8 @@ import org.springframework.transaction.annotation.Transactional; import org.springframework.util.Assert; +import lombok.extern.slf4j.Slf4j; + /** * @author Lars Helge Overland */ @@ -532,7 +534,7 @@ private List getColumnForOrgUnitDataElement(DataElement da .name((dataElement.getUid() + OU_GEOMETRY_COL_SUFFIX)) .dimensionType(AnalyticsDimensionType.DYNAMIC) .dataType(GEOMETRY) - .selectExpression(getOrgUnitSelectSubquery("geometry", dataElement)) + .selectExpression(getOrgUnitSelectSubquery(dataElement, "geometry")) .indexType(IndexType.GIST) .build()); } @@ -542,7 +544,7 @@ private List getColumnForOrgUnitDataElement(DataElement da .name((dataElement.getUid() + OU_NAME_COL_SUFFIX)) .dimensionType(AnalyticsDimensionType.DYNAMIC) .dataType(TEXT) - .selectExpression(getOrgUnitSelectSubquery("name", dataElement)) + .selectExpression(getOrgUnitSelectSubquery(dataElement, "name")) .skipIndex(SKIP) .build()); @@ -552,11 +554,11 @@ private List getColumnForOrgUnitDataElement(DataElement da /** * Returns a org unit select query. * - * @param column the column name. * @param dataElement the {@link DataElement}. + * @param column the column name. * @return an org unit select query. */ - private String getOrgUnitSelectSubquery(String column, DataElement dataElement) { + private String getOrgUnitSelectSubquery(DataElement dataElement, String column) { String format = """ (select ou.${column} from ${organisationunit} ou \ From c4b677519cdbdf1cc9e4bd64424e5697af6d76a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 10 Dec 2024 17:59:55 +0100 Subject: [PATCH 095/170] fix: Update code --- .../table/AbstractEventJdbcTableManager.java | 1 + .../JdbcEnrollmentAnalyticsTableManager.java | 1 - .../table/JdbcEventAnalyticsTableManager.java | 4 +- .../org/hisp/dhis/system/grid/ListGrid.java | 44 ++++++++++--------- 4 files changed, 26 insertions(+), 24 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index c6eb428842c1..6dc4d8c43b02 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -31,6 +31,7 @@ import static org.hisp.dhis.analytics.util.AnalyticsUtils.getColumnType; import static org.hisp.dhis.db.model.DataType.GEOMETRY; import static org.hisp.dhis.db.model.DataType.TEXT; + import java.util.ArrayList; import java.util.List; import java.util.Map; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java index 5ac39cc8f5b7..a57db9c8e68c 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java @@ -34,7 +34,6 @@ import java.util.Date; import java.util.List; import java.util.Map; - import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.AnalyticsTableType; import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index b545b7d893ed..8b66f85e24ea 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -50,7 +50,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; - +import lombok.extern.slf4j.Slf4j; import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.AnalyticsTableType; import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; @@ -89,8 +89,6 @@ import org.springframework.transaction.annotation.Transactional; import org.springframework.util.Assert; -import lombok.extern.slf4j.Slf4j; - /** * @author Lars Helge Overland */ diff --git a/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/grid/ListGrid.java b/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/grid/ListGrid.java index 17d4eb7c8e89..f6858bf40692 100644 --- a/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/grid/ListGrid.java +++ b/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/grid/ListGrid.java @@ -31,6 +31,11 @@ import static org.hisp.dhis.common.ValueType.getValueTypeFromSqlType; import static org.hisp.dhis.common.collection.CollectionUtils.mapToList; import static org.hisp.dhis.feedback.ErrorCode.E7230; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.google.common.collect.Iterables; import java.io.Serializable; import java.sql.ResultSet; import java.sql.ResultSetMetaData; @@ -47,6 +52,10 @@ import java.util.Set; import java.util.TreeMap; import java.util.regex.Pattern; +import lombok.AccessLevel; +import lombok.Getter; +import net.sf.jasperreports.engine.JRException; +import net.sf.jasperreports.engine.JRField; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.Validate; import org.apache.commons.math3.stat.regression.SimpleRegression; @@ -63,14 +72,6 @@ import org.hisp.dhis.system.util.MathUtils; import org.springframework.jdbc.support.rowset.SqlRowSet; import org.springframework.jdbc.support.rowset.SqlRowSetMetaData; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import com.google.common.collect.Iterables; -import lombok.AccessLevel; -import lombok.Getter; -import net.sf.jasperreports.engine.JRException; -import net.sf.jasperreports.engine.JRField; /** * @author Lars Helge Overland @@ -501,9 +502,10 @@ public Grid addColumn(List columnValues) { int currentColumnIndex = 0; if (grid.size() != columnValues.size()) { - throw new IllegalStateException(String.format( - "Number of column values (%d) is not equal to number of rows (%d)", - columnValues.size(), grid.size())); + throw new IllegalStateException( + String.format( + "Number of column values (%d) is not equal to number of rows (%d)", + columnValues.size(), grid.size())); } for (int i = 0; i < grid.size(); i++) { @@ -521,9 +523,10 @@ public Grid addColumn(int columnIndex, List columnValues) { int currentColumnIndex = 0; if (grid.size() != columnValues.size()) { - throw new IllegalStateException(String.format( - "Number of column values (%d) is not equal to number of rows (%d)", - columnValues.size(), grid.size())); + throw new IllegalStateException( + String.format( + "Number of column values (%d) is not equal to number of rows (%d)", + columnValues.size(), grid.size())); } for (int i = 0; i < grid.size(); i++) { @@ -799,7 +802,7 @@ public Grid substituteMetaData(Map metaDataMap) { GridHeader header = headers.get(colIndex); // Header - + Object headerMetaName = metaDataMap.get(header.getName()); if (headerMetaName != null) { @@ -1094,7 +1097,7 @@ public void repositionColumns(List columnIndexes) { } // Reposition columns in the row context structure - + Map> orderedRowContext = new HashMap<>(); for (Map.Entry> rowContextEntry : rowContext.entrySet()) { @@ -1107,7 +1110,7 @@ public void repositionColumns(List columnIndexes) { key -> { if (numberRegex.matcher(key).matches()) { // Reindexing of columns - + orderedRowContextItems.put( columnIndexes.get(Integer.parseInt(key)).toString(), ctxItem.get(key)); } @@ -1144,9 +1147,10 @@ private void verifyGridState() { for (List row : grid) { if (rowLength != null && rowLength != row.size()) { - throw new IllegalStateException(String.format( - "Grid rows do not have the same number of cells, previous: %d, this: %d, at row: %d", - rowLength, row.size(), rowPos)); + throw new IllegalStateException( + String.format( + "Grid rows do not have the same number of cells, previous: %d, this: %d, at row: %d", + rowLength, row.size(), rowPos)); } rowPos++; From 2bec9de2477aee656c6cffcc7623b796ed74ec44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 10 Dec 2024 18:44:38 +0100 Subject: [PATCH 096/170] fix: Update code --- .../dhis/analytics/table/AbstractEventJdbcTableManager.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 6dc4d8c43b02..8a98bd7bb0b0 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -49,7 +49,6 @@ import org.hisp.dhis.common.IdentifiableObjectManager; import org.hisp.dhis.common.ValueType; import org.hisp.dhis.dataapproval.DataApprovalLevelService; -import org.hisp.dhis.dataelement.DataElement; import org.hisp.dhis.db.model.DataType; import org.hisp.dhis.db.model.IndexType; import org.hisp.dhis.db.sql.SqlBuilder; @@ -258,8 +257,8 @@ private String getValueColumn(TrackedEntityAttribute attribute) { /** * Returns a org unit select query. * + * @param attribute the {@link TrackedEntityAttribute}. * @param column the column name. - * @param dataElement the {@link DataElement}. * @return an org unit select query. */ private String getOrgUnitSelectSubquery(TrackedEntityAttribute attribute, String column) { From b76b93048395946e053197caed229bd6b5681aa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 10 Dec 2024 19:49:21 +0100 Subject: [PATCH 097/170] fix: Update code --- .../table/JdbcEventAnalyticsTableManagerTest.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java index 9a9db09b4f99..11198d1e70f3 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java @@ -646,13 +646,18 @@ void verifyTeiTypeOrgUnitFetchesOuNameWhenPopulatingEventAnalyticsTable() { String ouNameQuery = String.format( """ - (select ou.name from "organisationunit" ou where ou.uid = \ - (select value from "trackedentityattributevalue" where trackedentityid=en.trackedentityid and \ - trackedentityattributeid=9999)) as %s""", + (select ou.name from "organisationunit" ou where ou.uid = %1$s.value) as %1$s""", + quote(tea.getUid())); + + String ouGeometryQuery = + String.format( + """ + (select ou.geometry from "organisationunit" ou where ou.uid = %1$s.value) as %1$s""", quote(tea.getUid())); assertThat(sql.getValue(), containsString(ouUidQuery)); assertThat(sql.getValue(), containsString(ouNameQuery)); + assertThat(sql.getValue(), containsString(ouGeometryQuery)); } @Test From 70fbb1f3472ded6ec04c86cfbc545d6597a0003b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 10 Dec 2024 19:53:34 +0100 Subject: [PATCH 098/170] fix: Update code --- .../JdbcEventAnalyticsTableManagerTest.java | 49 ++++++++++--------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java index 11198d1e70f3..15e1eb3b9a6d 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java @@ -501,11 +501,17 @@ void verifyGetTableWithTrackedEntityAttribute() { eventdatavalues #>> '{deabcdefghZ, value}' as "deabcdefghZ\""""; String aliasTeaUid = "%s.value"; - String aliasTea1 = - """ - (select %s from "organisationunit" ou where ou.uid = \ - (select value from "trackedentityattributevalue" where trackedentityid=en.trackedentityid \ - and trackedentityattributeid=%d)) as "%s\""""; + String ouGeometryQuery = + String.format( + """ + (select ou.geometry from "organisationunit" ou where ou.uid = %1$s.value) as %1$s""", + quote(tea1.getUid())); + + String ouNameQuery = + String.format( + """ + (select ou.name from "organisationunit" ou where ou.uid = %1$s.value) as %1$s""", + quote(tea1.getUid())); AnalyticsTableUpdateParams params = AnalyticsTableUpdateParams.newBuilder() @@ -535,17 +541,9 @@ void verifyGetTableWithTrackedEntityAttribute() { .addColumn(d1.getUid(), TEXT, toSelectExpression(aliasD1, d1.getUid()), Skip.SKIP) .addColumn(tea1.getUid(), TEXT, String.format(aliasTeaUid, quote(tea1.getUid()))) // Org unit geometry column - .addColumn( - tea1.getUid() + "_geom", - GEOMETRY, - String.format(aliasTea1, "ou.geometry", tea1.getId(), tea1.getUid()), - IndexType.GIST) + .addColumn(tea1.getUid() + "_geom", GEOMETRY, ouGeometryQuery, IndexType.GIST) // Org unit name column - .addColumn( - tea1.getUid() + "_name", - TEXT, - String.format(aliasTea1, "ou.name", tea1.getId(), tea1.getUid()), - Skip.SKIP) + .addColumn(tea1.getUid() + "_name", TEXT, ouNameQuery, Skip.SKIP) .withDefaultColumns(EventAnalyticsColumn.getColumns(sqlBuilder)) .build() .verify(); @@ -643,21 +641,21 @@ void verifyTeiTypeOrgUnitFetchesOuNameWhenPopulatingEventAnalyticsTable() { String ouUidQuery = String.format("%s.value", quote(tea.getUid())); - String ouNameQuery = + String ouGeometryQuery = String.format( """ - (select ou.name from "organisationunit" ou where ou.uid = %1$s.value) as %1$s""", + (select ou.geometry from "organisationunit" ou where ou.uid = %1$s.value) as %1$s""", quote(tea.getUid())); - String ouGeometryQuery = + String ouNameQuery = String.format( """ - (select ou.geometry from "organisationunit" ou where ou.uid = %1$s.value) as %1$s""", + (select ou.name from "organisationunit" ou where ou.uid = %1$s.value) as %1$s""", quote(tea.getUid())); assertThat(sql.getValue(), containsString(ouUidQuery)); - assertThat(sql.getValue(), containsString(ouNameQuery)); assertThat(sql.getValue(), containsString(ouGeometryQuery)); + assertThat(sql.getValue(), containsString(ouNameQuery)); } @Test @@ -937,15 +935,20 @@ void verifyTeaTypeOrgUnitFetchesOuNameWhenPopulatingEventAnalyticsTable() { String ouUidQuery = String.format("%s.value", quote(tea.getUid())); + String ouGeometryQuery = + String.format( + """ + (select ou.geometry from "organisationunit" ou where ou.uid = %1$s.value) as %1$s""", + quote(tea.getUid())); + String ouNameQuery = String.format( """ - (select ou.name from "organisationunit" ou where ou.uid = \ - (select value from "trackedentityattributevalue" where trackedentityid=en.trackedentityid \ - and trackedentityattributeid=9999)) as %s""", + (select ou.name from "organisationunit" ou where ou.uid = %1$s.value) as %1$s""", quote(tea.getUid())); assertThat(sql.getValue(), containsString(ouUidQuery)); + assertThat(sql.getValue(), containsString(ouGeometryQuery)); assertThat(sql.getValue(), containsString(ouNameQuery)); } From 9c4143c8ad9730e20b609f3d8ddc47401938d030 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 10 Dec 2024 20:04:45 +0100 Subject: [PATCH 099/170] fix: Update code --- .../dhis/analytics/table/JdbcEventAnalyticsTableManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 8b66f85e24ea..bde55a83d977 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -346,7 +346,7 @@ left join analytics_rs_dateperiodstructure dps on cast(${eventDateExpression} as inner join analytics_rs_categorystructure acs on ev.attributeoptioncomboid=acs.categoryoptioncomboid \ ${attributeJoinClause}\ where ev.lastupdated < '${startTime}' ${partitionClause} \ - and pr.programid=${programId} \ + and pr.programid = ${programId} \ and ev.organisationunitid is not null \ and (${eventDateExpression}) is not null \ and (ougs.startdate is null or dps.monthstartdate=ougs.startdate) \ From 8747d39fc2b41ccb0a1ba77bc679e58136faafed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 10 Dec 2024 22:18:33 +0100 Subject: [PATCH 100/170] fix: Update code --- ...EntityEventsAnalyticsTableManagerTest.java | 76 +++++++++---------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManagerTest.java index bcc61a915cad..4cb4bdf9f195 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManagerTest.java @@ -114,7 +114,7 @@ void testTableName() { } @Test - void testPopulate() { + void testPopulateTable() { TrackedEntityType tet = mock(TrackedEntityType.class); when(tet.getUid()).thenReturn("tetUid"); when(trackedEntityTypeService.getAllTrackedEntityType()).thenReturn(List.of(tet)); @@ -135,46 +135,46 @@ void testPopulate() { String subQuery = """ - (select json_object_agg(l2.keys, l2.datavalue) as value - from ( - select l1.uid, - l1.keys, - json_strip_nulls(json_build_object( - 'value', l1.eventdatavalues -> l1.keys ->> 'value', - 'created', l1.eventdatavalues -> l1.keys ->> 'created', - 'storedBy', l1.eventdatavalues -> l1.keys ->> 'storedBy', - 'lastUpdated', l1.eventdatavalues -> l1.keys ->> 'lastUpdated', - 'providedElsewhere', l1.eventdatavalues -> l1.keys -> 'providedElsewhere', - 'value_name', (select ou.name - from organisationunit ou - where ou.uid = l1.eventdatavalues -> l1.keys ->> 'value'), - 'value_code', (select ou.code - from organisationunit ou - where ou.uid = l1.eventdatavalues -> l1.keys ->> 'value'))) as datavalue - from (select inner_evt.*, jsonb_object_keys(inner_evt.eventdatavalues) keys - from event inner_evt) as l1) as l2 - where l2.uid = ev.uid - group by l2.uid)::jsonb - """; + (select json_object_agg(l2.keys, l2.datavalue) as value + from ( + select l1.uid, + l1.keys, + json_strip_nulls(json_build_object( + 'value', l1.eventdatavalues -> l1.keys ->> 'value', + 'created', l1.eventdatavalues -> l1.keys ->> 'created', + 'storedBy', l1.eventdatavalues -> l1.keys ->> 'storedBy', + 'lastUpdated', l1.eventdatavalues -> l1.keys ->> 'lastUpdated', + 'providedElsewhere', l1.eventdatavalues -> l1.keys -> 'providedElsewhere', + 'value_name', (select ou.name + from organisationunit ou + where ou.uid = l1.eventdatavalues -> l1.keys ->> 'value'), + 'value_code', (select ou.code + from organisationunit ou + where ou.uid = l1.eventdatavalues -> l1.keys ->> 'value'))) as datavalue + from (select inner_evt.*, jsonb_object_keys(inner_evt.eventdatavalues) keys + from event inner_evt) as l1) as l2 + where l2.uid = ev.uid + group by l2.uid)::jsonb + """; String expectedSql = """ - insert into analytics_te_event_tetuid_temp ("trackedentity","program","enrollment","programstage","event","occurreddate","lastupdated","created", - "scheduleddate","status","uidlevel1","uidlevel2","uidlevel3","uidlevel4","ou","ouname","oucode","oulevel","eventdatavalues","eventgeometry", - "evlongitude","evlatitude","ounamehierarchy") select distinct te.uid,p.uid,en.uid,ps.uid,ev.uid,ev.occurreddate,ev.lastupdated, - ev.created,ev.scheduleddate,ev.status,ous.uidlevel1,ous.uidlevel2,ous.uidlevel3,ous.uidlevel4,ous.organisationunituid,ous.name,ous.code,ous.level, - %s, - ev.geometry,case when 'POINT' = GeometryType(ev.geometry) then ST_X(ev.geometry) end,case when 'POINT' = GeometryType(ev.geometry) then ST_Y(ev.geometry) end,concat_ws(' / ',) as ounamehierarchy - from "event" ev inner join "enrollment" en on en.enrollmentid=ev.enrollmentid - and en.deleted = false inner join "trackedentity" te on te.trackedentityid=en.trackedentityid - and te.deleted = false and te.trackedentitytypeid = 0 - and te.lastupdated < '2019-08-01T00:00:00' left join "programstage" ps on ev.programstageid=ps.programstageid - left join "program" p on ps.programid=p.programid - left join analytics_rs_orgunitstructure ous on ev.organisationunitid=ous.organisationunitid - where ev.status in ('COMPLETED','ACTIVE','SCHEDULE') - and (CASE WHEN 'SCHEDULE' = ev.status THEN ev.scheduleddate ELSE ev.occurreddate END) >= 'null' - and (CASE WHEN 'SCHEDULE' = ev.status THEN ev.scheduleddate ELSE ev.occurreddate END) < 'null' - and ev.deleted = false""" + insert into analytics_te_event_tetuid_temp ("trackedentity","program","enrollment","programstage","event","occurreddate","lastupdated","created", + "scheduleddate","status","uidlevel1","uidlevel2","uidlevel3","uidlevel4","ou","ouname","oucode","oulevel","eventdatavalues","eventgeometry", + "evlongitude","evlatitude","ounamehierarchy") select distinct te.uid,p.uid,en.uid,ps.uid,ev.uid,ev.occurreddate,ev.lastupdated, + ev.created,ev.scheduleddate,ev.status,ous.uidlevel1,ous.uidlevel2,ous.uidlevel3,ous.uidlevel4,ous.organisationunituid,ous.name,ous.code,ous.level, + %s, + ev.geometry,case when 'POINT' = GeometryType(ev.geometry) then ST_X(ev.geometry) end,case when 'POINT' = GeometryType(ev.geometry) then ST_Y(ev.geometry) end,concat_ws(' / ',) as ounamehierarchy + from "event" ev inner join "enrollment" en on en.enrollmentid=ev.enrollmentid + and en.deleted = false inner join "trackedentity" te on te.trackedentityid=en.trackedentityid + and te.deleted = false and te.trackedentitytypeid = 0 + and te.lastupdated < '2019-08-01T00:00:00' left join "programstage" ps on ev.programstageid=ps.programstageid + left join "program" p on ps.programid=p.programid + left join analytics_rs_orgunitstructure ous on ev.organisationunitid=ous.organisationunitid + where ev.status in ('COMPLETED','ACTIVE','SCHEDULE') + and (CASE WHEN 'SCHEDULE' = ev.status THEN ev.scheduleddate ELSE ev.occurreddate END) >= 'null' + and (CASE WHEN 'SCHEDULE' = ev.status THEN ev.scheduleddate ELSE ev.occurreddate END) < 'null' + and ev.deleted = false""" .formatted(subQuery); ArgumentCaptor sqlCaptor = ArgumentCaptor.forClass(String.class); From 311bb4f2dd826b0590dca046b9bf2b7142ba38a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 10 Dec 2024 22:19:43 +0100 Subject: [PATCH 101/170] fix: Update code --- .../dhis/analytics/table/AnalyticsTableServiceTest.java | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java index 91389524f5de..3322599ee643 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java @@ -30,7 +30,6 @@ import static org.hisp.dhis.db.model.DataType.DOUBLE; import static org.hisp.dhis.db.model.DataType.TEXT; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.when; import java.util.List; @@ -43,7 +42,6 @@ import org.hisp.dhis.setting.SystemSettings; import org.hisp.dhis.setting.SystemSettingsProvider; import org.joda.time.DateTime; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; @@ -63,11 +61,6 @@ class AnalyticsTableServiceTest { @InjectMocks private DefaultAnalyticsTableService tableService; - @BeforeEach - void setUp() { - lenient().when(settingsProvider.getCurrentSettings()).thenReturn(settings); - } - @Test void testGetTablePartitions() { when(sqlBuilder.supportsDeclarativePartitioning()).thenReturn(false); @@ -104,6 +97,7 @@ void testGetTablePartitions() { @Test void testGetParallelJobsA() { + when(settingsProvider.getCurrentSettings()).thenReturn(settings); when(settings.getParallelJobsInAnalyticsTableExport()).thenReturn(1); assertEquals(1, tableService.getParallelJobs()); @@ -111,6 +105,7 @@ void testGetParallelJobsA() { @Test void testGetParallelJobsB() { + when(settingsProvider.getCurrentSettings()).thenReturn(settings); when(settings.getParallelJobsInAnalyticsTableExport()).thenReturn(-1); when(settings.getDatabaseServerCpus()).thenReturn(8); From 62eb6645daf4a9af28790fbd369394450cbf8488 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 10 Dec 2024 22:30:06 +0100 Subject: [PATCH 102/170] fix: Update code --- .../table/JdbcAnalyticsTableManagerTest.java | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerTest.java index 76a6eb4bd460..7c968b08511a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerTest.java @@ -35,7 +35,6 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -68,6 +67,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentMatchers; +import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.Spy; @@ -82,38 +82,38 @@ @MockitoSettings(strictness = Strictness.LENIENT) @ExtendWith(MockitoExtension.class) class JdbcAnalyticsTableManagerTest { + @Mock private IdentifiableObjectManager idObjectManager; + + @Mock private OrganisationUnitService organisationUnitService; + + @Mock private CategoryService categoryService; + @Mock private SystemSettingsProvider settingsProvider; @Mock private SystemSettings settings; + @Mock private DataApprovalLevelService dataApprovalLevelService; + + @Mock private ResourceTableService resourceTableService; + + @Mock private AnalyticsTableHookService analyticsTableHookService; + + @Mock private PartitionManager partitionManager; + @Mock private JdbcTemplate jdbcTemplate; @Mock private AnalyticsTableSettings analyticsTableSettings; @Mock private PeriodDataProvider periodDataProvider; - @Spy private final SqlBuilder sqlBuilder = new PostgreSqlBuilder(); + @Spy private SqlBuilder sqlBuilder = new PostgreSqlBuilder(); - private JdbcAnalyticsTableManager subject; + @InjectMocks private JdbcAnalyticsTableManager subject; @BeforeEach public void setUp() { when(settingsProvider.getCurrentSettings()).thenReturn(settings); when(settings.getLastSuccessfulResourceTablesUpdate()).thenReturn(new Date(0L)); - subject = - new JdbcAnalyticsTableManager( - mock(IdentifiableObjectManager.class), - mock(OrganisationUnitService.class), - mock(CategoryService.class), - settingsProvider, - mock(DataApprovalLevelService.class), - mock(ResourceTableService.class), - mock(AnalyticsTableHookService.class), - mock(PartitionManager.class), - jdbcTemplate, - analyticsTableSettings, - periodDataProvider, - sqlBuilder); } @Test From 6a4c3b196e9b5c065486cbf8cbe50cf566a8f0b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 10 Dec 2024 22:32:33 +0100 Subject: [PATCH 103/170] fix: Update code --- ...bcEnrollmentAnalyticsTableManagerTest.java | 36 ++++++++++--------- .../analytics/table/PartitionUtilsTest.java | 10 +++--- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManagerTest.java index f2f2d642cf42..81f150f5c395 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManagerTest.java @@ -68,7 +68,9 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; +import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.Spy; import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.jdbc.core.JdbcTemplate; @@ -79,15 +81,31 @@ class JdbcEnrollmentAnalyticsTableManagerTest { @Mock private IdentifiableObjectManager idObjectManager; + @Mock private OrganisationUnitService organisationUnitService; + + @Mock private CategoryService categoryService; + + @Mock private SystemSettingsProvider settingsProvider; + + @Mock private SystemSettings settings; + + @Mock private DataApprovalLevelService dataApprovalLevelService; + + @Mock private ResourceTableService resourceTableService; + + @Mock private AnalyticsTableHookService analyticsTableHookService; + + @Mock private PartitionManager partitionManager; + @Mock private JdbcTemplate jdbcTemplate; @Mock private AnalyticsTableSettings analyticsTableSettings; @Mock private PeriodDataProvider periodDataProvider; - private final SqlBuilder sqlBuilder = new PostgreSqlBuilder(); + @Spy private SqlBuilder sqlBuilder = new PostgreSqlBuilder(); - private JdbcEnrollmentAnalyticsTableManager subject; + @InjectMocks private JdbcEnrollmentAnalyticsTableManager subject; private static final Date START_TIME = new DateTime(2019, 8, 1, 0, 0).toDate(); @@ -96,20 +114,6 @@ public void setUp() { when(analyticsTableSettings.isSpatialSupport()).thenReturn(true); SystemSettingsProvider settingsProvider = mock(SystemSettingsProvider.class); lenient().when(settingsProvider.getCurrentSettings()).thenReturn(SystemSettings.of(Map.of())); - subject = - new JdbcEnrollmentAnalyticsTableManager( - idObjectManager, - mock(OrganisationUnitService.class), - mock(CategoryService.class), - settingsProvider, - mock(DataApprovalLevelService.class), - mock(ResourceTableService.class), - mock(AnalyticsTableHookService.class), - mock(PartitionManager.class), - jdbcTemplate, - analyticsTableSettings, - periodDataProvider, - sqlBuilder); } @Test diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/PartitionUtilsTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/PartitionUtilsTest.java index cbd1947a91e8..20265191bce3 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/PartitionUtilsTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/PartitionUtilsTest.java @@ -47,13 +47,11 @@ * @author Lars Helge Overland */ class PartitionUtilsTest { - private PeriodType quarterly = new QuarterlyPeriodType(); + private final PeriodType quarterly = new QuarterlyPeriodType(); - private Period q1 = quarterly.createPeriod(new DateTime(2018, 7, 1, 0, 0).toDate()); - - private Period q2 = quarterly.createPeriod(new DateTime(2018, 10, 1, 0, 0).toDate()); - - private Period q3 = quarterly.createPeriod(new DateTime(2019, 1, 1, 0, 0).toDate()); + private final Period q1 = quarterly.createPeriod(new DateTime(2018, 7, 1, 0, 0).toDate()); + private final Period q2 = quarterly.createPeriod(new DateTime(2018, 10, 1, 0, 0).toDate()); + private final Period q3 = quarterly.createPeriod(new DateTime(2019, 1, 1, 0, 0).toDate()); @Test void testGetPartitions() { From f51f4c3280f62b35649178e7e3864e250a717671 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 10 Dec 2024 23:05:27 +0100 Subject: [PATCH 104/170] fix: Update code --- .../table/JdbcEnrollmentAnalyticsTableManagerTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManagerTest.java index 81f150f5c395..4d7463b8fb6e 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManagerTest.java @@ -36,7 +36,6 @@ import static org.hisp.dhis.test.TestBase.createTrackedEntityAttribute; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.mockito.Mockito.lenient; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -112,7 +111,6 @@ class JdbcEnrollmentAnalyticsTableManagerTest { @BeforeEach public void setUp() { when(analyticsTableSettings.isSpatialSupport()).thenReturn(true); - SystemSettingsProvider settingsProvider = mock(SystemSettingsProvider.class); lenient().when(settingsProvider.getCurrentSettings()).thenReturn(SystemSettings.of(Map.of())); } From 7f9e59d1a0b1d330d7940e0d92b9b17e5b78eb7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 10 Dec 2024 23:07:17 +0100 Subject: [PATCH 105/170] fix: Update code --- .../trackedentity/TrackedEntityQueryRequestValidatorTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/trackedentity/TrackedEntityQueryRequestValidatorTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/trackedentity/TrackedEntityQueryRequestValidatorTest.java index 1e1b37f8629f..4437c074a2cd 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/trackedentity/TrackedEntityQueryRequestValidatorTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/trackedentity/TrackedEntityQueryRequestValidatorTest.java @@ -54,7 +54,7 @@ void testValidateWhenTrackedEntityTypeIsInvalid() { () -> trackedEntityQueryRequestValidator.validate(trackedEntityRequestParams)); assertEquals( - "Invalid UID `" + teiUid + "` for property `trackedEntityType`", exception.getMessage()); + "Invalid UID '" + teiUid + "' for property 'trackedEntityType'", exception.getMessage()); } @Test @@ -69,6 +69,6 @@ void testValidateWhenNoTrackedEntityType() { IllegalQueryException.class, () -> trackedEntityQueryRequestValidator.validate(trackedEntityRequestParams)); - assertEquals("Invalid UID `null` for property `trackedEntityType`", exception.getMessage()); + assertEquals("Invalid UID 'null' for property 'trackedEntityType'", exception.getMessage()); } } From fd72ecc4d9f92cad8212128df1afced6c591ed83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 10 Dec 2024 23:07:34 +0100 Subject: [PATCH 106/170] fix: Update code --- .../trackedentity/TrackedEntityQueryRequestValidatorTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/trackedentity/TrackedEntityQueryRequestValidatorTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/trackedentity/TrackedEntityQueryRequestValidatorTest.java index 4437c074a2cd..1e1b37f8629f 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/trackedentity/TrackedEntityQueryRequestValidatorTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/trackedentity/TrackedEntityQueryRequestValidatorTest.java @@ -54,7 +54,7 @@ void testValidateWhenTrackedEntityTypeIsInvalid() { () -> trackedEntityQueryRequestValidator.validate(trackedEntityRequestParams)); assertEquals( - "Invalid UID '" + teiUid + "' for property 'trackedEntityType'", exception.getMessage()); + "Invalid UID `" + teiUid + "` for property `trackedEntityType`", exception.getMessage()); } @Test @@ -69,6 +69,6 @@ void testValidateWhenNoTrackedEntityType() { IllegalQueryException.class, () -> trackedEntityQueryRequestValidator.validate(trackedEntityRequestParams)); - assertEquals("Invalid UID 'null' for property 'trackedEntityType'", exception.getMessage()); + assertEquals("Invalid UID `null` for property `trackedEntityType`", exception.getMessage()); } } From a0257a32b3c7fdab8705635febecb15a08b8b186 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 11 Dec 2024 09:34:45 +0100 Subject: [PATCH 107/170] fix: Update code --- ...ntityEnrollmentsAnalyticsTableManager.java | 24 ------------------- ...ckedEntityEventsAnalyticsTableManager.java | 24 ------------------- .../dhis/db/sql/ClickHouseSqlBuilder.java | 5 ++++ .../org/hisp/dhis/db/sql/DorisSqlBuilder.java | 5 ++++ .../hisp/dhis/db/sql/PostgreSqlBuilder.java | 5 ++++ .../java/org/hisp/dhis/db/sql/SqlBuilder.java | 5 ++++ 6 files changed, 20 insertions(+), 48 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEnrollmentsAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEnrollmentsAnalyticsTableManager.java index 70428872148d..b99aa7af1502 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEnrollmentsAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEnrollmentsAnalyticsTableManager.java @@ -115,30 +115,6 @@ public class JdbcTrackedEntityEnrollmentsAnalyticsTableManager extends AbstractJ .dataType(VARCHAR_50) .selectExpression("en.status") .build(), - AnalyticsTableColumn.builder() - .name("uidlevel1") - .dataType(CHARACTER_11) - .nullable(NULL) - .selectExpression("ous.uidlevel1") - .build(), - AnalyticsTableColumn.builder() - .name("uidlevel2") - .dataType(CHARACTER_11) - .nullable(NULL) - .selectExpression("ous.uidlevel2") - .build(), - AnalyticsTableColumn.builder() - .name("uidlevel3") - .dataType(CHARACTER_11) - .nullable(NULL) - .selectExpression("ous.uidlevel3") - .build(), - AnalyticsTableColumn.builder() - .name("uidlevel4") - .dataType(CHARACTER_11) - .nullable(NULL) - .selectExpression("ous.uidlevel4") - .build(), AnalyticsTableColumn.builder() .name("ou") .dataType(CHARACTER_11) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManager.java index a89213becd23..9b96121fff2e 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManager.java @@ -143,30 +143,6 @@ public class JdbcTrackedEntityEventsAnalyticsTableManager extends AbstractJdbcTa .dataType(VARCHAR_50) .selectExpression("ev.status") .build(), - AnalyticsTableColumn.builder() - .name("uidlevel1") - .dataType(CHARACTER_11) - .nullable(NULL) - .selectExpression("ous.uidlevel1") - .build(), - AnalyticsTableColumn.builder() - .name("uidlevel2") - .dataType(CHARACTER_11) - .nullable(NULL) - .selectExpression("ous.uidlevel2") - .build(), - AnalyticsTableColumn.builder() - .name("uidlevel3") - .dataType(CHARACTER_11) - .nullable(NULL) - .selectExpression("ous.uidlevel3") - .build(), - AnalyticsTableColumn.builder() - .name("uidlevel4") - .dataType(CHARACTER_11) - .nullable(NULL) - .selectExpression("ous.uidlevel4") - .build(), AnalyticsTableColumn.builder() .name("ou") .dataType(CHARACTER_11) diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java index f763eace89e7..6381be607075 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java @@ -164,6 +164,11 @@ public boolean supportsVacuum() { return false; } + @Override + public boolean supportsCorrelatedSubquery() { + return false; + } + @Override public boolean requiresIndexesForAnalytics() { return false; diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java index e4414841175f..48bfe6bc31b8 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java @@ -166,6 +166,11 @@ public boolean supportsVacuum() { return false; } + @Override + public boolean supportsCorrelatedSubquery() { + return true; + } + @Override public boolean requiresIndexesForAnalytics() { return false; diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgreSqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgreSqlBuilder.java index f28259ce271b..92a14270f8cc 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgreSqlBuilder.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgreSqlBuilder.java @@ -185,6 +185,11 @@ public boolean supportsVacuum() { return true; } + @Override + public boolean supportsCorrelatedSubquery() { + return true; + } + @Override public boolean requiresIndexesForAnalytics() { return true; diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java index 78bbd2d866a4..07016f4c6e8d 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java @@ -174,6 +174,11 @@ public interface SqlBuilder { */ boolean supportsVacuum(); + /** + * @return true if the DBMS supports corrected subqueries. + */ + boolean supportsCorrelatedSubquery(); + /** * @return true if the DBMS requires indexes for analytics tables for performance. */ From 5ef9f34570cab8d142ec9e28016707735b67e030 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 11 Dec 2024 09:37:59 +0100 Subject: [PATCH 108/170] fix: Update code --- .../JdbcTrackedEntityEnrollmentsAnalyticsTableManager.java | 1 + .../table/JdbcTrackedEntityEventsAnalyticsTableManager.java | 1 + .../table/JdbcTrackedEntityEventsAnalyticsTableManagerTest.java | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEnrollmentsAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEnrollmentsAnalyticsTableManager.java index b99aa7af1502..a00e34eb392b 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEnrollmentsAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEnrollmentsAnalyticsTableManager.java @@ -201,6 +201,7 @@ private List getColumns() { List columns = new ArrayList<>(); columns.addAll(getFixedCols()); columns.add(getOrganisationUnitNameHierarchyColumn()); + columns.addAll(getOrganisationUnitLevelColumns()); if (sqlBuilder.supportsDeclarativePartitioning()) { columns.add(getPartitionColumn()); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManager.java index 9b96121fff2e..b3968442378b 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManager.java @@ -343,6 +343,7 @@ private List getColumns() { } columns.add(getOrganisationUnitNameHierarchyColumn()); + columns.addAll(getOrganisationUnitLevelColumns()); return columns; } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManagerTest.java index 4cb4bdf9f195..6d1806fed5c2 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManagerTest.java @@ -162,7 +162,7 @@ void testPopulateTable() { insert into analytics_te_event_tetuid_temp ("trackedentity","program","enrollment","programstage","event","occurreddate","lastupdated","created", "scheduleddate","status","uidlevel1","uidlevel2","uidlevel3","uidlevel4","ou","ouname","oucode","oulevel","eventdatavalues","eventgeometry", "evlongitude","evlatitude","ounamehierarchy") select distinct te.uid,p.uid,en.uid,ps.uid,ev.uid,ev.occurreddate,ev.lastupdated, - ev.created,ev.scheduleddate,ev.status,ous.uidlevel1,ous.uidlevel2,ous.uidlevel3,ous.uidlevel4,ous.organisationunituid,ous.name,ous.code,ous.level, + ev.created,ev.scheduleddate,ev.status,ous.organisationunituid,ous.name,ous.code,ous.level, %s, ev.geometry,case when 'POINT' = GeometryType(ev.geometry) then ST_X(ev.geometry) end,case when 'POINT' = GeometryType(ev.geometry) then ST_Y(ev.geometry) end,concat_ws(' / ',) as ounamehierarchy from "event" ev inner join "enrollment" en on en.enrollmentid=ev.enrollmentid From 536a9504deecc24fe85120c3d9675ecc38ff8f1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 11 Dec 2024 09:45:32 +0100 Subject: [PATCH 109/170] fix: Update code --- ...dbcTrackedEntityAnalyticsTableManager.java | 2 +- .../dhis/analytics/util/AnalyticsUtils.java | 30 ------------------- ...EntityEventsAnalyticsTableManagerTest.java | 2 +- .../analytics/util/AnalyticsUtilsTest.java | 8 ----- 4 files changed, 2 insertions(+), 40 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java index 4a483d156d37..e99a46c56562 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java @@ -267,7 +267,7 @@ private String castBasedOnType(ValueType valueType, String columnName) { " cast(${columnName} as ${type})", Map.of("columnName", columnName, "type", sqlBuilder.dataTypeTimestamp())); } - if (valueType.isGeo() && isSpatialSupport() && sqlBuilder.supportsGeospatialData()) { + if (valueType.isGeo() && isSpatialSupport()) { return replace( """ \s ST_GeomFromGeoJSON('{"type":"Point", "coordinates":' || (${columnName}) || ', diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsUtils.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsUtils.java index cf73c25f349d..b61b7a82bbaa 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsUtils.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsUtils.java @@ -27,7 +27,6 @@ */ package org.hisp.dhis.analytics.util; -import static org.apache.commons.lang3.StringUtils.EMPTY; import static org.hisp.dhis.common.DataDimensionItem.DATA_DIM_TYPE_CLASS_MAP; import static org.hisp.dhis.common.DimensionalObject.ATTRIBUTEOPTIONCOMBO_DIM_ID; import static org.hisp.dhis.common.DimensionalObject.CATEGORYOPTIONCOMBO_DIM_ID; @@ -1202,33 +1201,4 @@ public static String replaceStringBetween( Matcher matcher = pattern.matcher(original); return matcher.replaceAll(startToken + replacement + endToken); } - - /** - * Returns a string containing closing parenthesis. The number of parenthesis is based on the - * number of missing closing parenthesis in the argument string. - * - *

Example: - * - *

{@code} input: "((( ))" -> output: ")" {@code} - * - * @param str a string. - * @return a String containing 0 or more "closing" parenthesis - */ - public static String getClosingParentheses(String str) { - if (StringUtils.isEmpty(str)) { - return EMPTY; - } - - int open = 0; - - for (int i = 0; i < str.length(); i++) { - if (str.charAt(i) == '(') { - open++; - } else if ((str.charAt(i) == ')') && open >= 1) { - open--; - } - } - - return StringUtils.repeat(")", open); - } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManagerTest.java index 6d1806fed5c2..57fce6c172ac 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManagerTest.java @@ -160,7 +160,7 @@ void testPopulateTable() { String expectedSql = """ insert into analytics_te_event_tetuid_temp ("trackedentity","program","enrollment","programstage","event","occurreddate","lastupdated","created", - "scheduleddate","status","uidlevel1","uidlevel2","uidlevel3","uidlevel4","ou","ouname","oucode","oulevel","eventdatavalues","eventgeometry", + "scheduleddate","status","ou","ouname","oucode","oulevel","eventdatavalues","eventgeometry", "evlongitude","evlatitude","ounamehierarchy") select distinct te.uid,p.uid,en.uid,ps.uid,ev.uid,ev.occurreddate,ev.lastupdated, ev.created,ev.scheduleddate,ev.status,ous.organisationunituid,ous.name,ous.code,ous.level, %s, diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/util/AnalyticsUtilsTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/util/AnalyticsUtilsTest.java index dbbb563aa2ac..d2b77d2fecbf 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/util/AnalyticsUtilsTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/util/AnalyticsUtilsTest.java @@ -739,14 +739,6 @@ void testGetEnumCaseSensitivity() { assertNull(EnumUtils.getEnum(Database.class, "postgresql")); } - @Test - void testGetClosingParentheses() { - assertEquals("", AnalyticsUtils.getClosingParentheses(null)); - assertEquals("", AnalyticsUtils.getClosingParentheses("")); - assertEquals(")", AnalyticsUtils.getClosingParentheses("from(select(select (*))")); - assertEquals("))", AnalyticsUtils.getClosingParentheses("((")); - } - @Test void whenUncategorizedSQLException_withTableNotExisting_thenThrowException() { SQLException sqlException = new SQLException("relation does not exist", "42P01"); From dcc92066031bb879f388ae11fd530f6b4e0a73f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 11 Dec 2024 09:49:35 +0100 Subject: [PATCH 110/170] fix: Update code --- ...dbcTrackedEntityAnalyticsTableManager.java | 41 +------------------ 1 file changed, 2 insertions(+), 39 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java index e99a46c56562..5ef9f9a562d4 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java @@ -35,7 +35,6 @@ import static org.hisp.dhis.analytics.util.AnalyticsUtils.getColumnType; import static org.hisp.dhis.analytics.util.DisplayNameUtils.getDisplayName; import static org.hisp.dhis.commons.util.TextUtils.removeLastComma; -import static org.hisp.dhis.commons.util.TextUtils.replace; import static org.hisp.dhis.db.model.DataType.BOOLEAN; import static org.hisp.dhis.db.model.DataType.CHARACTER_11; import static org.hisp.dhis.db.model.DataType.DOUBLE; @@ -65,7 +64,6 @@ import org.hisp.dhis.analytics.table.setting.AnalyticsTableSettings; import org.hisp.dhis.category.CategoryService; import org.hisp.dhis.common.IdentifiableObjectManager; -import org.hisp.dhis.common.ValueType; import org.hisp.dhis.dataapproval.DataApprovalLevelService; import org.hisp.dhis.db.model.IndexType; import org.hisp.dhis.db.model.Logged; @@ -85,7 +83,7 @@ import org.springframework.transaction.annotation.Transactional; @Component("org.hisp.dhis.analytics.TrackedEntityAnalyticsTableManager") -public class JdbcTrackedEntityAnalyticsTableManager extends AbstractJdbcTableManager { +public class JdbcTrackedEntityAnalyticsTableManager extends AbstractEventJdbcTableManager { private static final String PROGRAMS_BY_TET_KEY = "programsByTetUid"; private static final String ALL_NON_CONFIDENTIAL_TET_ATTRIBUTES = @@ -215,7 +213,7 @@ private List getColumns( .name(tea.getUid()) .dataType(getColumnType(tea.getValueType(), isSpatialSupport())) .selectExpression( - castBasedOnType(tea.getValueType(), quote(tea.getUid()) + ".value")) + getColumnExpression(tea.getValueType(), quote(tea.getUid()) + ".value")) .build()) .toList()); @@ -242,41 +240,6 @@ private Stream getAllTrackedEntityAttributes( return getAllTrackedEntityAttributesByEntityType(trackedEntityType); } - /** - * Returns the select clause, potentially with a cast statement, based on the given value type. - * - * @param valueType the value type to represent as database column type. - */ - private String castBasedOnType(ValueType valueType, String columnName) { - if (valueType.isDecimal()) { - - return replace( - " cast(${columnName} as ${type})", - Map.of("columnName", columnName, "type", sqlBuilder.dataTypeDouble())); - } - if (valueType.isInteger()) { - return replace(" cast(${columnName} as bigint)", Map.of("columnName", columnName)); - } - if (valueType.isBoolean()) { - return replace( - " case when ${columnName} = 'true' then 1 when ${columnName} = 'false' then 0 end ", - Map.of("columnName", columnName)); - } - if (valueType.isDate()) { - return replace( - " cast(${columnName} as ${type})", - Map.of("columnName", columnName, "type", sqlBuilder.dataTypeTimestamp())); - } - if (valueType.isGeo() && isSpatialSupport()) { - return replace( - """ - \s ST_GeomFromGeoJSON('{"type":"Point", "coordinates":' || (${columnName}) || ', - "crs":{"type":"name", "properties":{"name":"EPSG:4326"}}}')""", - Map.of("columnName", columnName)); - } - return columnName; - } - /** * Returns all {@link TrackedEntityAttribute} for the given {@link TrackedEntityType} and * programs. From 394f2bad024ded0530e6fd3a03cccd36e80dc0b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 11 Dec 2024 10:43:21 +0100 Subject: [PATCH 111/170] fix: Update code --- .../table/JdbcAnalyticsTableManager.java | 27 +++++----- .../table/JdbcCompletenessTableManager.java | 17 +++--- .../JdbcCompletenessTargetTableManager.java | 13 ++--- .../table/JdbcOrgUnitTargetTableManager.java | 13 ++--- .../JdbcOwnershipAnalyticsTableManager.java | 12 ++--- ...dbcTrackedEntityAnalyticsTableManager.java | 26 ++++------ ...ckedEntityEventsAnalyticsTableManager.java | 52 +++++++++---------- .../JdbcValidationResultTableManager.java | 13 ++--- 8 files changed, 70 insertions(+), 103 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java index 359bba2ce1d2..de1396c68e30 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java @@ -340,22 +340,19 @@ private void populateTable( StringBuilder sql = new StringBuilder(replace("insert into ${tableName} (", Map.of("tableName", tableName))); - List dimensions = partition.getMasterTable().getDimensionColumns(); List columns = partition.getMasterTable().getAnalyticsTableColumns(); + List dimensions = partition.getMasterTable().getDimensionColumns(); - for (AnalyticsTableColumn col : columns) { - sql.append(quote(col.getName()) + ","); - } + sql.append(toCommaSeparated(columns, col -> quote(col.getName()))); - sql = TextUtils.removeLastComma(sql).append(") select "); + sql.append(") select "); - for (AnalyticsTableColumn col : dimensions) { - sql.append(col.getSelectExpression() + ","); - } + sql.append(toCommaSeparated(dimensions, AnalyticsTableColumn::getSelectExpression)); - sql.append( - replaceQualify( - """ + sql.append(",") + .append( + replaceQualify( + """ ${approvalSelectExpression} \ as approvallevel, \ ${valueExpression} * ps.daysno as daysxvalue, \ @@ -372,10 +369,10 @@ private void populateTable( inner join analytics_rs_categorystructure acs on dv.attributeoptioncomboid=acs.categoryoptioncomboid \ inner join analytics_rs_categoryoptioncomboname aon on dv.attributeoptioncomboid=aon.categoryoptioncomboid \ inner join analytics_rs_categoryoptioncomboname con on dv.categoryoptioncomboid=con.categoryoptioncomboid\s""", - Map.of( - "approvalSelectExpression", approvalSelectExpression, - "valueExpression", valueExpression, - "textValueExpression", textValueExpression))); + Map.of( + "approvalSelectExpression", approvalSelectExpression, + "valueExpression", valueExpression, + "textValueExpression", textValueExpression))); if (!params.isSkipOutliers()) { sql.append(getOutliersJoinStatement()); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java index fe98336a13f9..cbcc46e9ab74 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java @@ -57,7 +57,6 @@ import org.hisp.dhis.analytics.table.setting.AnalyticsTableSettings; import org.hisp.dhis.category.CategoryService; import org.hisp.dhis.common.IdentifiableObjectManager; -import org.hisp.dhis.commons.util.TextUtils; import org.hisp.dhis.dataapproval.DataApprovalLevelService; import org.hisp.dhis.db.sql.SqlBuilder; import org.hisp.dhis.organisationunit.OrganisationUnitService; @@ -196,21 +195,17 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti String tableName = partition.getName(); String partitionClause = getPartitionClause(partition); - String sql = "insert into " + tableName + " ("; - List columns = partition.getMasterTable().getAnalyticsTableColumns(); - for (AnalyticsTableColumn col : columns) { - sql += quote(col.getName()) + ","; - } + String sql = "insert into " + tableName + " ("; - sql = TextUtils.removeLastComma(sql) + ") select "; + sql += toCommaSeparated(columns, col -> quote(col.getName())); - for (AnalyticsTableColumn col : columns) { - sql += col.getSelectExpression() + ","; - } + sql += ") select "; + + sql += toCommaSeparated(columns, AnalyticsTableColumn::getSelectExpression); - sql = TextUtils.removeLastComma(sql) + " "; + sql += " "; // Database legacy fix diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTargetTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTargetTableManager.java index 4b74d8ccee2a..7d75de0ab2e8 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTargetTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTargetTableManager.java @@ -48,7 +48,6 @@ import org.hisp.dhis.analytics.table.setting.AnalyticsTableSettings; import org.hisp.dhis.category.CategoryService; import org.hisp.dhis.common.IdentifiableObjectManager; -import org.hisp.dhis.commons.util.TextUtils; import org.hisp.dhis.dataapproval.DataApprovalLevelService; import org.hisp.dhis.db.model.Logged; import org.hisp.dhis.db.sql.SqlBuilder; @@ -163,17 +162,13 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti List columns = partition.getMasterTable().getAnalyticsTableColumns(); - for (AnalyticsTableColumn col : columns) { - sql += quote(col.getName()) + ","; - } + sql += toCommaSeparated(columns, col -> quote(col.getName())); - sql = TextUtils.removeLastComma(sql) + ") select "; + sql += ") select "; - for (AnalyticsTableColumn col : columns) { - sql += col.getSelectExpression() + ","; - } + sql += toCommaSeparated(columns, AnalyticsTableColumn::getSelectExpression); - sql = TextUtils.removeLastComma(sql) + " "; + sql += " "; sql += qualifyVariables( diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOrgUnitTargetTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOrgUnitTargetTableManager.java index 5874ec70477f..d5a3826f6ad8 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOrgUnitTargetTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOrgUnitTargetTableManager.java @@ -49,7 +49,6 @@ import org.hisp.dhis.analytics.table.setting.AnalyticsTableSettings; import org.hisp.dhis.category.CategoryService; import org.hisp.dhis.common.IdentifiableObjectManager; -import org.hisp.dhis.commons.util.TextUtils; import org.hisp.dhis.dataapproval.DataApprovalLevelService; import org.hisp.dhis.db.model.Logged; import org.hisp.dhis.db.sql.SqlBuilder; @@ -138,17 +137,13 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti List columns = partition.getMasterTable().getAnalyticsTableColumns(); - for (AnalyticsTableColumn col : columns) { - sql += quote(col.getName()) + ","; - } + sql += toCommaSeparated(columns, col -> quote(col.getName())); - sql = TextUtils.removeLastComma(sql) + ") select "; + sql += ") select "; - for (AnalyticsTableColumn col : columns) { - sql += col.getSelectExpression() + ","; - } + sql += toCommaSeparated(columns, AnalyticsTableColumn::getSelectExpression); - sql = TextUtils.removeLastComma(sql) + " "; + sql += " "; sql += qualifyVariables( diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java index 2d3b77847e19..543e3c2e4f5c 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java @@ -239,15 +239,13 @@ private void populateOwnershipTableInternal(AnalyticsTablePartition partition, S * @return a SQL select query. */ private String getInputSql(Program program) { - StringBuilder sb = new StringBuilder("select "); + List columns = getColumns(); - for (AnalyticsTableColumn col : getColumns()) { - sb.append(col.getSelectExpression()).append(","); - } + StringBuilder sql = new StringBuilder("select "); - sb.deleteCharAt(sb.length() - 1); // Remove the final ','. + sql.append(toCommaSeparated(columns, AnalyticsTableColumn::getSelectExpression)); - sb.append( + sql.append( replaceQualify( """ \sfrom (\ @@ -272,7 +270,7 @@ and exists (\ "historyTableId", HISTORY_TABLE_ID, "trackedEntityOwnTableId", TRACKED_ENTITY_OWN_TABLE_ID, "programId", String.valueOf(program.getId())))); - return sb.toString(); + return sql.toString(); } private Map getRowMap(List columnNames, ResultSet resultSet) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java index 5ef9f9a562d4..6fa553f43054 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java @@ -34,7 +34,6 @@ import static org.hisp.dhis.analytics.table.JdbcEventAnalyticsTableManager.EXPORTABLE_EVENT_STATUSES; import static org.hisp.dhis.analytics.util.AnalyticsUtils.getColumnType; import static org.hisp.dhis.analytics.util.DisplayNameUtils.getDisplayName; -import static org.hisp.dhis.commons.util.TextUtils.removeLastComma; import static org.hisp.dhis.db.model.DataType.BOOLEAN; import static org.hisp.dhis.db.model.DataType.CHARACTER_11; import static org.hisp.dhis.db.model.DataType.DOUBLE; @@ -296,26 +295,21 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti StringBuilder sql = new StringBuilder("insert into " + tableName + " ("); - for (AnalyticsTableColumn col : columns) { - sql.append(quote(col.getName()) + ","); - } + sql.append(toCommaSeparated(columns, col -> quote(col.getName()))); - removeLastComma(sql).append(") select "); + sql.append(") select "); - for (AnalyticsTableColumn col : columns) { - sql.append(col.getSelectExpression() + ","); - } + sql.append(toCommaSeparated(columns, AnalyticsTableColumn::getSelectExpression)); TrackedEntityType trackedEntityType = partition.getMasterTable().getTrackedEntityType(); - removeLastComma(sql) - .append( - replaceQualify( - """ - \sfrom ${trackedentity} te \ - left join analytics_rs_orgunitstructure ous on te.organisationunitid=ous.organisationunitid \ - left join analytics_rs_organisationunitgroupsetstructure ougs on te.organisationunitid=ougs.organisationunitid""", - Map.of())); + sql.append( + replaceQualify( + """ + \sfrom ${trackedentity} te \ + left join analytics_rs_orgunitstructure ous on te.organisationunitid=ous.organisationunitid \ + left join analytics_rs_organisationunitgroupsetstructure ougs on te.organisationunitid=ougs.organisationunitid""", + Map.of())); ((List) params.getExtraParam(trackedEntityType.getUid(), ALL_NON_CONFIDENTIAL_TET_ATTRIBUTES)) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManager.java index b3968442378b..2361238c6314 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManager.java @@ -34,7 +34,6 @@ import static org.hisp.dhis.analytics.table.util.PartitionUtils.getStartDate; import static org.hisp.dhis.commons.util.TextUtils.emptyIfTrue; import static org.hisp.dhis.commons.util.TextUtils.format; -import static org.hisp.dhis.commons.util.TextUtils.removeLastComma; import static org.hisp.dhis.commons.util.TextUtils.replace; import static org.hisp.dhis.db.model.DataType.CHARACTER_11; import static org.hisp.dhis.db.model.DataType.DOUBLE; @@ -369,35 +368,34 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti StringBuilder sql = new StringBuilder("insert into " + tableName + " ("); - for (AnalyticsTableColumn col : columns) { - sql.append(quote(col.getName()) + ","); - } + sql.append(toCommaSeparated(columns, col -> quote(col.getName()))); - removeLastComma(sql).append(") select distinct "); + sql.append(") select distinct "); - for (AnalyticsTableColumn col : columns) { - sql.append(col.getSelectExpression() + ","); - } + sql.append(toCommaSeparated(columns, AnalyticsTableColumn::getSelectExpression)); - removeLastComma(sql) - .append( - replaceQualify( - """ - \s from ${event} ev \ - inner join ${enrollment} en on en.enrollmentid=ev.enrollmentid and en.deleted = false \ - inner join ${trackedentity} te on te.trackedentityid=en.trackedentityid \ - and te.deleted = false and te.trackedentitytypeid = ${tetId} and te.lastupdated < '${startTime}' \ - left join ${programstage} ps on ev.programstageid=ps.programstageid \ - left join ${program} p on ps.programid=p.programid \ - left join analytics_rs_orgunitstructure ous on ev.organisationunitid=ous.organisationunitid \ - where ev.status in (${statuses}) \ - ${partitionClause} \ - and ev.deleted = false\s""", - Map.of( - "tetId", String.valueOf(masterTable.getTrackedEntityType().getId()), - "startTime", toLongDate(params.getStartTime()), - "statuses", join(",", EXPORTABLE_EVENT_STATUSES), - "partitionClause", partitionClause))); + sql.append( + replaceQualify( + """ + \s from ${event} ev \ + inner join ${enrollment} en on en.enrollmentid=ev.enrollmentid and en.deleted = false \ + inner join ${trackedentity} te on te.trackedentityid=en.trackedentityid \ + and te.deleted = false and te.trackedentitytypeid = ${tetId} and te.lastupdated < '${startTime}' \ + left join ${programstage} ps on ev.programstageid=ps.programstageid \ + left join ${program} p on ps.programid=p.programid \ + left join analytics_rs_orgunitstructure ous on ev.organisationunitid=ous.organisationunitid \ + where ev.status in (${statuses}) \ + ${partitionClause} \ + and ev.deleted = false\s""", + Map.of( + "tetId", + String.valueOf(masterTable.getTrackedEntityType().getId()), + "startTime", + toLongDate(params.getStartTime()), + "statuses", + join(",", EXPORTABLE_EVENT_STATUSES), + "partitionClause", + partitionClause))); invokeTimeAndLog(sql.toString(), "Populating table: '{}'", tableName); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcValidationResultTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcValidationResultTableManager.java index 26614fdc3c12..25e0d02a1d81 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcValidationResultTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcValidationResultTableManager.java @@ -30,7 +30,6 @@ import static org.hisp.dhis.analytics.table.model.AnalyticsValueType.FACT; import static org.hisp.dhis.commons.util.TextUtils.emptyIfTrue; import static org.hisp.dhis.commons.util.TextUtils.format; -import static org.hisp.dhis.commons.util.TextUtils.removeLastComma; import static org.hisp.dhis.db.model.DataType.CHARACTER_11; import static org.hisp.dhis.db.model.DataType.DATE; import static org.hisp.dhis.db.model.DataType.INTEGER; @@ -168,17 +167,13 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti List columns = partition.getMasterTable().getAnalyticsTableColumns(); - for (AnalyticsTableColumn col : columns) { - sql += quote(col.getName()) + ","; - } + sql += toCommaSeparated(columns, col -> quote(col.getName())); - sql = removeLastComma(sql) + ") select "; + sql += ") select "; - for (AnalyticsTableColumn col : columns) { - sql += col.getSelectExpression() + ","; - } + sql += toCommaSeparated(columns, AnalyticsTableColumn::getSelectExpression); - sql = removeLastComma(sql) + " "; + sql += " "; // Database legacy fix From 324f8e98d16c92260bbaecad9855a029f624399e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 11 Dec 2024 10:46:17 +0100 Subject: [PATCH 112/170] fix: Update code --- .../JdbcCompletenessTargetTableManager.java | 7 +-- .../table/JdbcOrgUnitTargetTableManager.java | 7 +-- ...dbcTrackedEntityAnalyticsTableManager.java | 3 -- ...ntityEnrollmentsAnalyticsTableManager.java | 53 ++++++++----------- .../JdbcValidationResultTableManager.java | 6 +-- 5 files changed, 27 insertions(+), 49 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTargetTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTargetTableManager.java index 7d75de0ab2e8..618a08419c35 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTargetTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTargetTableManager.java @@ -158,16 +158,13 @@ protected List getPartitionChecks(Integer year, Date endDate) { public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTablePartition partition) { String tableName = partition.getName(); - String sql = "insert into " + tableName + " ("; - List columns = partition.getMasterTable().getAnalyticsTableColumns(); - sql += toCommaSeparated(columns, col -> quote(col.getName())); + String sql = "insert into " + tableName + " ("; + sql += toCommaSeparated(columns, col -> quote(col.getName())); sql += ") select "; - sql += toCommaSeparated(columns, AnalyticsTableColumn::getSelectExpression); - sql += " "; sql += diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOrgUnitTargetTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOrgUnitTargetTableManager.java index d5a3826f6ad8..fccec23f0279 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOrgUnitTargetTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOrgUnitTargetTableManager.java @@ -133,16 +133,13 @@ protected List getPartitionChecks(Integer year, Date endDate) { public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTablePartition partition) { String tableName = partition.getName(); - String sql = replace("insert into ${tableName} (", Map.of("tableName", quote(tableName))); - List columns = partition.getMasterTable().getAnalyticsTableColumns(); - sql += toCommaSeparated(columns, col -> quote(col.getName())); + String sql = replace("insert into ${tableName} (", Map.of("tableName", quote(tableName))); + sql += toCommaSeparated(columns, col -> quote(col.getName())); sql += ") select "; - sql += toCommaSeparated(columns, AnalyticsTableColumn::getSelectExpression); - sql += " "; sql += diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java index 6fa553f43054..3507f4ad42f2 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java @@ -294,11 +294,8 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti List columns = partition.getMasterTable().getAnalyticsTableColumns(); StringBuilder sql = new StringBuilder("insert into " + tableName + " ("); - sql.append(toCommaSeparated(columns, col -> quote(col.getName()))); - sql.append(") select "); - sql.append(toCommaSeparated(columns, AnalyticsTableColumn::getSelectExpression)); TrackedEntityType trackedEntityType = partition.getMasterTable().getTrackedEntityType(); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEnrollmentsAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEnrollmentsAnalyticsTableManager.java index a00e34eb392b..36e2569c1f68 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEnrollmentsAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEnrollmentsAnalyticsTableManager.java @@ -31,7 +31,6 @@ import static java.lang.String.valueOf; import static org.hisp.dhis.analytics.AnalyticsTableType.TRACKED_ENTITY_INSTANCE_ENROLLMENTS; import static org.hisp.dhis.analytics.table.JdbcEventAnalyticsTableManager.EXPORTABLE_EVENT_STATUSES; -import static org.hisp.dhis.commons.util.TextUtils.removeLastComma; import static org.hisp.dhis.db.model.DataType.CHARACTER_11; import static org.hisp.dhis.db.model.DataType.DOUBLE; import static org.hisp.dhis.db.model.DataType.GEOMETRY; @@ -226,37 +225,29 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti List columns = partition.getMasterTable().getAnalyticsTableColumns(); StringBuilder sql = new StringBuilder("insert into " + tableName + " ("); + sql.append(toCommaSeparated(columns, col -> quote(col.getName()))); + sql.append(") select "); + sql.append(toCommaSeparated(columns, AnalyticsTableColumn::getSelectExpression)); - for (AnalyticsTableColumn col : columns) { - sql.append(quote(col.getName()) + ","); - } - - removeLastComma(sql).append(") select "); - - for (AnalyticsTableColumn col : columns) { - sql.append(col.getSelectExpression() + ","); - } - - removeLastComma(sql) - .append( - replaceQualify( - """ - \sfrom ${enrollment} en \ - inner join ${trackedentity} te on en.trackedentityid=te.trackedentityid \ - and te.deleted = false and te.trackedentitytypeid = ${trackedEntityTypeId} \ - and te.lastupdated < '${startTime}' \ - left join ${program} p on en.programid=p.programid \ - left join analytics_rs_orgunitstructure ous on en.organisationunitid=ous.organisationunitid \ - where exists (select 1 from ${event} ev where ev.deleted = false \ - and ev.enrollmentid = en.enrollmentid \ - and ev.status in (${statuses})) \ - and en.occurreddate is not null \ - and en.deleted = false\s""", - Map.of( - "trackedEntityTypeId", - valueOf(partition.getMasterTable().getTrackedEntityType().getId()), - "startTime", toLongDate(params.getStartTime()), - "statuses", join(",", EXPORTABLE_EVENT_STATUSES)))); + sql.append( + replaceQualify( + """ + \sfrom ${enrollment} en \ + inner join ${trackedentity} te on en.trackedentityid=te.trackedentityid \ + and te.deleted = false and te.trackedentitytypeid = ${trackedEntityTypeId} \ + and te.lastupdated < '${startTime}' \ + left join ${program} p on en.programid=p.programid \ + left join analytics_rs_orgunitstructure ous on en.organisationunitid=ous.organisationunitid \ + where exists (select 1 from ${event} ev where ev.deleted = false \ + and ev.enrollmentid = en.enrollmentid \ + and ev.status in (${statuses})) \ + and en.occurreddate is not null \ + and en.deleted = false\s""", + Map.of( + "trackedEntityTypeId", + valueOf(partition.getMasterTable().getTrackedEntityType().getId()), + "startTime", toLongDate(params.getStartTime()), + "statuses", join(",", EXPORTABLE_EVENT_STATUSES)))); invokeTimeAndLog(sql.toString(), "Populating table: '{}'", tableName); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcValidationResultTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcValidationResultTableManager.java index 25e0d02a1d81..e6409edcd11f 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcValidationResultTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcValidationResultTableManager.java @@ -163,16 +163,12 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti String tableName = partition.getName(); String partitionClause = getPartitionClause(partition); - String sql = "insert into " + tableName + " ("; - List columns = partition.getMasterTable().getAnalyticsTableColumns(); + String sql = "insert into " + tableName + " ("; sql += toCommaSeparated(columns, col -> quote(col.getName())); - sql += ") select "; - sql += toCommaSeparated(columns, AnalyticsTableColumn::getSelectExpression); - sql += " "; // Database legacy fix From c9ca1f2928eb8fc487daeed3e925b50a2e3a9f24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 11 Dec 2024 10:47:37 +0100 Subject: [PATCH 113/170] fix: Update code --- .../table/JdbcAnalyticsTableManager.java | 50 +++++++++---------- .../table/JdbcCompletenessTableManager.java | 16 ++---- .../JdbcCompletenessTargetTableManager.java | 1 - 3 files changed, 29 insertions(+), 38 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java index de1396c68e30..31d587e51392 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java @@ -344,35 +344,33 @@ private void populateTable( List dimensions = partition.getMasterTable().getDimensionColumns(); sql.append(toCommaSeparated(columns, col -> quote(col.getName()))); - sql.append(") select "); - sql.append(toCommaSeparated(dimensions, AnalyticsTableColumn::getSelectExpression)); + sql.append(","); - sql.append(",") - .append( - replaceQualify( - """ - ${approvalSelectExpression} \ - as approvallevel, \ - ${valueExpression} * ps.daysno as daysxvalue, \ - ps.daysno as daysno, \ - ${valueExpression} as value, \ - ${textValueExpression} as textvalue \ - from ${datavalue} dv \ - inner join analytics_rs_periodstructure ps on dv.periodid=ps.periodid \ - inner join analytics_rs_dataelementstructure des on dv.dataelementid=des.dataelementid \ - inner join analytics_rs_dataelementgroupsetstructure degs on dv.dataelementid=degs.dataelementid \ - inner join analytics_rs_orgunitstructure ous on dv.sourceid=ous.organisationunitid \ - inner join analytics_rs_organisationunitgroupsetstructure ougs on dv.sourceid=ougs.organisationunitid \ - inner join analytics_rs_categorystructure dcs on dv.categoryoptioncomboid=dcs.categoryoptioncomboid \ - inner join analytics_rs_categorystructure acs on dv.attributeoptioncomboid=acs.categoryoptioncomboid \ - inner join analytics_rs_categoryoptioncomboname aon on dv.attributeoptioncomboid=aon.categoryoptioncomboid \ - inner join analytics_rs_categoryoptioncomboname con on dv.categoryoptioncomboid=con.categoryoptioncomboid\s""", - Map.of( - "approvalSelectExpression", approvalSelectExpression, - "valueExpression", valueExpression, - "textValueExpression", textValueExpression))); + sql.append( + replaceQualify( + """ + ${approvalSelectExpression} \ + as approvallevel, \ + ${valueExpression} * ps.daysno as daysxvalue, \ + ps.daysno as daysno, \ + ${valueExpression} as value, \ + ${textValueExpression} as textvalue \ + from ${datavalue} dv \ + inner join analytics_rs_periodstructure ps on dv.periodid=ps.periodid \ + inner join analytics_rs_dataelementstructure des on dv.dataelementid=des.dataelementid \ + inner join analytics_rs_dataelementgroupsetstructure degs on dv.dataelementid=degs.dataelementid \ + inner join analytics_rs_orgunitstructure ous on dv.sourceid=ous.organisationunitid \ + inner join analytics_rs_organisationunitgroupsetstructure ougs on dv.sourceid=ougs.organisationunitid \ + inner join analytics_rs_categorystructure dcs on dv.categoryoptioncomboid=dcs.categoryoptioncomboid \ + inner join analytics_rs_categorystructure acs on dv.attributeoptioncomboid=acs.categoryoptioncomboid \ + inner join analytics_rs_categoryoptioncomboname aon on dv.attributeoptioncomboid=aon.categoryoptioncomboid \ + inner join analytics_rs_categoryoptioncomboname con on dv.categoryoptioncomboid=con.categoryoptioncomboid\s""", + Map.of( + "approvalSelectExpression", approvalSelectExpression, + "valueExpression", valueExpression, + "textValueExpression", textValueExpression))); if (!params.isSkipOutliers()) { sql.append(getOutliersJoinStatement()); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java index cbcc46e9ab74..ba1d569a99b7 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java @@ -150,11 +150,11 @@ public boolean hasUpdatedLatestData(Date startDate, Date endDate) { String sql = replaceQualify( """ - select cdr.datasetid \ - from ${completedatasetregistration} cdr \ - where cdr.lastupdated >= '${startDate}' \ - and cdr.lastupdated < '${endDate}' \ - limit 1;""", + select cdr.datasetid \ + from ${completedatasetregistration} cdr \ + where cdr.lastupdated >= '${startDate}' \ + and cdr.lastupdated < '${endDate}' \ + limit 1;""", Map.of("startDate", toLongDate(startDate), "endDate", toLongDate(endDate))); return !jdbcTemplate.queryForList(sql).isEmpty(); @@ -198,17 +198,11 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti List columns = partition.getMasterTable().getAnalyticsTableColumns(); String sql = "insert into " + tableName + " ("; - sql += toCommaSeparated(columns, col -> quote(col.getName())); - sql += ") select "; - sql += toCommaSeparated(columns, AnalyticsTableColumn::getSelectExpression); - sql += " "; - // Database legacy fix - sql = sql.replace("organisationunitid", "sourceid"); sql += diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTargetTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTargetTableManager.java index 618a08419c35..ed32291a7083 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTargetTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTargetTableManager.java @@ -161,7 +161,6 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti List columns = partition.getMasterTable().getAnalyticsTableColumns(); String sql = "insert into " + tableName + " ("; - sql += toCommaSeparated(columns, col -> quote(col.getName())); sql += ") select "; sql += toCommaSeparated(columns, AnalyticsTableColumn::getSelectExpression); From 9ee1cb6e6cf009543b6427b423baa4531d12211e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 11 Dec 2024 10:49:14 +0100 Subject: [PATCH 114/170] fix: Update code --- .../table/JdbcOrgUnitTargetTableManager.java | 1 - .../JdbcOwnershipAnalyticsTableManager.java | 1 - ...ckedEntityEventsAnalyticsTableManager.java | 24 +++++++++---------- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOrgUnitTargetTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOrgUnitTargetTableManager.java index fccec23f0279..9515e9eac449 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOrgUnitTargetTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOrgUnitTargetTableManager.java @@ -136,7 +136,6 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti List columns = partition.getMasterTable().getAnalyticsTableColumns(); String sql = replace("insert into ${tableName} (", Map.of("tableName", quote(tableName))); - sql += toCommaSeparated(columns, col -> quote(col.getName())); sql += ") select "; sql += toCommaSeparated(columns, AnalyticsTableColumn::getSelectExpression); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java index 543e3c2e4f5c..50bf484bddb9 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java @@ -242,7 +242,6 @@ private String getInputSql(Program program) { List columns = getColumns(); StringBuilder sql = new StringBuilder("select "); - sql.append(toCommaSeparated(columns, AnalyticsTableColumn::getSelectExpression)); sql.append( diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManager.java index 2361238c6314..4101d6adc5e8 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManager.java @@ -367,26 +367,24 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti sqlBuilder.supportsDeclarativePartitioning() ? "" : getPartitionClause(partition); StringBuilder sql = new StringBuilder("insert into " + tableName + " ("); - sql.append(toCommaSeparated(columns, col -> quote(col.getName()))); - sql.append(") select distinct "); - sql.append(toCommaSeparated(columns, AnalyticsTableColumn::getSelectExpression)); + sql.append(" "); sql.append( replaceQualify( """ - \s from ${event} ev \ - inner join ${enrollment} en on en.enrollmentid=ev.enrollmentid and en.deleted = false \ - inner join ${trackedentity} te on te.trackedentityid=en.trackedentityid \ - and te.deleted = false and te.trackedentitytypeid = ${tetId} and te.lastupdated < '${startTime}' \ - left join ${programstage} ps on ev.programstageid=ps.programstageid \ - left join ${program} p on ps.programid=p.programid \ - left join analytics_rs_orgunitstructure ous on ev.organisationunitid=ous.organisationunitid \ - where ev.status in (${statuses}) \ - ${partitionClause} \ - and ev.deleted = false\s""", + from ${event} ev \ + inner join ${enrollment} en on en.enrollmentid=ev.enrollmentid and en.deleted = false \ + inner join ${trackedentity} te on te.trackedentityid=en.trackedentityid \ + and te.deleted = false and te.trackedentitytypeid = ${tetId} and te.lastupdated < '${startTime}' \ + left join ${programstage} ps on ev.programstageid=ps.programstageid \ + left join ${program} p on ps.programid=p.programid \ + left join analytics_rs_orgunitstructure ous on ev.organisationunitid=ous.organisationunitid \ + where ev.status in (${statuses}) \ + ${partitionClause} \ + and ev.deleted = false\s""", Map.of( "tetId", String.valueOf(masterTable.getTrackedEntityType().getId()), From 39c08cf98f558b529c270d6f3917c33e7a0a7c24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 11 Dec 2024 10:52:29 +0100 Subject: [PATCH 115/170] fix: Update code --- .../JdbcTrackedEntityEnrollmentsAnalyticsTableManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEnrollmentsAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEnrollmentsAnalyticsTableManager.java index 36e2569c1f68..31d7c936eda9 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEnrollmentsAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEnrollmentsAnalyticsTableManager.java @@ -221,6 +221,7 @@ protected List getPartitionChecks(Integer year, Date endDate) { @Override public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTablePartition partition) { String tableName = partition.getName(); + long tetId = partition.getMasterTable().getTrackedEntityType().getId(); List columns = partition.getMasterTable().getAnalyticsTableColumns(); @@ -244,8 +245,7 @@ and ev.status in (${statuses})) \ and en.occurreddate is not null \ and en.deleted = false\s""", Map.of( - "trackedEntityTypeId", - valueOf(partition.getMasterTable().getTrackedEntityType().getId()), + "trackedEntityTypeId", valueOf(tetId), "startTime", toLongDate(params.getStartTime()), "statuses", join(",", EXPORTABLE_EVENT_STATUSES)))); From 318e862e2d6e22d93b4be1ac7ef9d962e0cef827 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 11 Dec 2024 10:53:57 +0100 Subject: [PATCH 116/170] fix: Update code --- ...dbcTrackedEntityEventsAnalyticsTableManager.java | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManager.java index 4101d6adc5e8..7f9aed298e6a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManager.java @@ -362,6 +362,7 @@ protected List getPartitionChecks(Integer year, Date endDate) { public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTablePartition partition) { AnalyticsTable masterTable = partition.getMasterTable(); String tableName = partition.getName(); + long tetId = masterTable.getTrackedEntityType().getId(); List columns = partition.getMasterTable().getAnalyticsTableColumns(); String partitionClause = sqlBuilder.supportsDeclarativePartitioning() ? "" : getPartitionClause(partition); @@ -386,14 +387,10 @@ where ev.status in (${statuses}) \ ${partitionClause} \ and ev.deleted = false\s""", Map.of( - "tetId", - String.valueOf(masterTable.getTrackedEntityType().getId()), - "startTime", - toLongDate(params.getStartTime()), - "statuses", - join(",", EXPORTABLE_EVENT_STATUSES), - "partitionClause", - partitionClause))); + "tetId", String.valueOf(tetId), + "startTime", toLongDate(params.getStartTime()), + "statuses", join(",", EXPORTABLE_EVENT_STATUSES), + "partitionClause", partitionClause))); invokeTimeAndLog(sql.toString(), "Populating table: '{}'", tableName); } From 0cfacb148cd7626e35cc2446624f18adbd09b97f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 11 Dec 2024 10:54:42 +0100 Subject: [PATCH 117/170] fix: Update code --- .../table/JdbcTrackedEntityAnalyticsTableManager.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java index 3507f4ad42f2..dc0e4030bcf0 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java @@ -303,9 +303,9 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti sql.append( replaceQualify( """ - \sfrom ${trackedentity} te \ - left join analytics_rs_orgunitstructure ous on te.organisationunitid=ous.organisationunitid \ - left join analytics_rs_organisationunitgroupsetstructure ougs on te.organisationunitid=ougs.organisationunitid""", + \sfrom ${trackedentity} te \ + left join analytics_rs_orgunitstructure ous on te.organisationunitid=ous.organisationunitid \ + left join analytics_rs_organisationunitgroupsetstructure ougs on te.organisationunitid=ougs.organisationunitid""", Map.of())); ((List) From 718d1be230853982631346614ee8ba34579d7ee8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 11 Dec 2024 12:40:30 +0100 Subject: [PATCH 118/170] fix: Update code --- .../table/JdbcTrackedEntityAnalyticsTableManager.java | 6 ------ .../JdbcTrackedEntityEnrollmentsAnalyticsTableManager.java | 5 +---- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java index dc0e4030bcf0..d302e6ee4a0c 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java @@ -325,12 +325,6 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti """ \swhere te.trackedentitytypeid = ${tetId} \ and te.lastupdated < '${startTime}' \ - and exists (select 1 from ${enrollment} en \ - where en.trackedentityid = te.trackedentityid \ - and exists (select 1 from ${event} ev \ - where ev.enrollmentid = en.enrollmentid \ - and ev.status in (${statuses}) \ - and ev.deleted = false)) \ and te.created is not null \ and te.deleted = false""", Map.of( diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEnrollmentsAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEnrollmentsAnalyticsTableManager.java index 31d7c936eda9..cc09a8cf81c5 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEnrollmentsAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEnrollmentsAnalyticsTableManager.java @@ -239,10 +239,7 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti and te.lastupdated < '${startTime}' \ left join ${program} p on en.programid=p.programid \ left join analytics_rs_orgunitstructure ous on en.organisationunitid=ous.organisationunitid \ - where exists (select 1 from ${event} ev where ev.deleted = false \ - and ev.enrollmentid = en.enrollmentid \ - and ev.status in (${statuses})) \ - and en.occurreddate is not null \ + where en.occurreddate is not null \ and en.deleted = false\s""", Map.of( "trackedEntityTypeId", valueOf(tetId), From c679653fb610a1bae8ebaf12656b3dfd37d5b58e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 11 Dec 2024 12:56:43 +0100 Subject: [PATCH 119/170] fix: Update code --- .../JdbcTrackedEntityEnrollmentsAnalyticsTableManager.java | 5 ----- .../table/JdbcTrackedEntityEventsAnalyticsTableManager.java | 5 ----- 2 files changed, 10 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEnrollmentsAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEnrollmentsAnalyticsTableManager.java index cc09a8cf81c5..d6cf52b6aba7 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEnrollmentsAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEnrollmentsAnalyticsTableManager.java @@ -171,11 +171,6 @@ public JdbcTrackedEntityEnrollmentsAnalyticsTableManager( this.trackedEntityTypeService = trackedEntityTypeService; } - /** - * Returns the {@link AnalyticsTableType} of analytics table which this manager handles. - * - * @return type of analytics table. - */ @Override public AnalyticsTableType getAnalyticsTableType() { return TRACKED_ENTITY_INSTANCE_ENROLLMENTS; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManager.java index 7f9aed298e6a..a67de2ad69f8 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManager.java @@ -203,11 +203,6 @@ public JdbcTrackedEntityEventsAnalyticsTableManager( this.analyticsSqlBuilder = analyticsSqlBuilder; } - /** - * Returns the {@link AnalyticsTableType} of analytics table which this manager handles. - * - * @return type of analytics table. - */ @Override public AnalyticsTableType getAnalyticsTableType() { return TRACKED_ENTITY_INSTANCE_EVENTS; From 221a551975311fc0de74660296b5593fb9879b91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 11 Dec 2024 14:24:10 +0100 Subject: [PATCH 120/170] fix: Update code --- .../table/AbstractEventJdbcTableManager.java | 8 ++--- .../table/JdbcEventAnalyticsTableManager.java | 29 ++++++++++++++++--- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 8a98bd7bb0b0..42b02d3f21db 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -31,7 +31,6 @@ import static org.hisp.dhis.analytics.util.AnalyticsUtils.getColumnType; import static org.hisp.dhis.db.model.DataType.GEOMETRY; import static org.hisp.dhis.db.model.DataType.TEXT; - import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -170,11 +169,8 @@ protected void populateTableInternal(AnalyticsTablePartition partition, String f String sql = "insert into " + tableName + " ("; sql += toCommaSeparated(columns, col -> quote(col.getName())); - sql += ") select "; - sql += toCommaSeparated(columns, AnalyticsTableColumn::getSelectExpression); - sql += " " + fromClause; invokeTimeAndLog(sql, "Populating table: '{}'", tableName); @@ -218,6 +214,10 @@ protected List getColumnForAttribute(TrackedEntityAttribut */ private List getColumnForOrgUnitAttribute( TrackedEntityAttribute attribute) { + if (!sqlBuilder.supportsCorrelatedSubquery()) { + return List.of(); + } + Validate.isTrue(attribute.getValueType().isOrganisationUnit()); List columns = new ArrayList<>(); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index bde55a83d977..ee2e795cdd97 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -41,7 +41,6 @@ import static org.hisp.dhis.system.util.MathUtils.NUMERIC_LENIENT_REGEXP; import static org.hisp.dhis.util.DateUtils.toLongDate; import static org.hisp.dhis.util.DateUtils.toMediumDate; - import java.time.Year; import java.util.ArrayList; import java.util.Collection; @@ -50,7 +49,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import lombok.extern.slf4j.Slf4j; import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.AnalyticsTableType; import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; @@ -88,6 +86,7 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.util.Assert; +import lombok.extern.slf4j.Slf4j; /** * @author Lars Helge Overland @@ -492,8 +491,7 @@ private List getColumnForDataElement( sqlBuilder.jsonExtractNested("eventdatavalues", dataElement.getUid(), "value"); String columnExpression = getColumnExpression(dataElement.getValueType(), jsonExpression); String dataFilterClause = getDataFilterClause(dataElement); - String selectExpression = - String.format("%s as %s", columnExpression, quote(dataElement.getUid())); + String selectExpression = getSelectExpression(dataElement, columnExpression); Skip skipIndex = skipIndex(dataElement.getValueType(), dataElement.hasOptionSet()); if (withLegendSet) { @@ -516,6 +514,17 @@ private List getColumnForDataElement( return columns; } + /** + * Retyrns a select expression. + * + * @param dataElement the {@link DataElement}. + * @param columnExpression the column expression. + * @return a select expression. + */ + private String getSelectExpression(DataElement dataElement, String columnExpression) { + return String.format("%s as %s", columnExpression, quote(dataElement.getUid())); + } + /** * Returns a list of columns. * @@ -524,6 +533,10 @@ private List getColumnForDataElement( * @return a list of {@link AnalyticsTableColumn}. */ private List getColumnForOrgUnitDataElement(DataElement dataElement) { + if (!sqlBuilder.supportsCorrelatedSubquery()) { + return List.of(); + } + List columns = new ArrayList<>(); if (isSpatialSupport()) { @@ -602,6 +615,10 @@ private List getAttributeColumns(Program program) { */ private List getColumnForAttributeWithLegendSet( TrackedEntityAttribute attribute) { + if (!sqlBuilder.supportsCorrelatedSubquery()) { + return List.of(); + } + String columnExpression = getColumnExpression(attribute.getValueType(), "value"); String numericClause = getNumericClause("value"); String query = @@ -646,6 +663,10 @@ private List getColumnForAttributeWithLegendSet( */ private List getColumnFromDataElementWithLegendSet( DataElement dataElement, String selectExpression, String dataFilterClause) { + if (!sqlBuilder.supportsCorrelatedSubquery()) { + return List.of(); + } + String query = """ (select l.uid from ${maplegend} l \ From 7b0af20a6340f1a6fb6be18aa195ab497d0faeef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 11 Dec 2024 14:27:01 +0100 Subject: [PATCH 121/170] fix: Update code --- .../table/AbstractEventJdbcTableManager.java | 3 ++- .../table/JdbcEventAnalyticsTableManager.java | 15 ++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 42b02d3f21db..cd7894fb23db 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -31,6 +31,7 @@ import static org.hisp.dhis.analytics.util.AnalyticsUtils.getColumnType; import static org.hisp.dhis.db.model.DataType.GEOMETRY; import static org.hisp.dhis.db.model.DataType.TEXT; + import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -217,7 +218,7 @@ private List getColumnForOrgUnitAttribute( if (!sqlBuilder.supportsCorrelatedSubquery()) { return List.of(); } - + Validate.isTrue(attribute.getValueType().isOrganisationUnit()); List columns = new ArrayList<>(); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index ee2e795cdd97..f5ae60d8af0a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -41,6 +41,7 @@ import static org.hisp.dhis.system.util.MathUtils.NUMERIC_LENIENT_REGEXP; import static org.hisp.dhis.util.DateUtils.toLongDate; import static org.hisp.dhis.util.DateUtils.toMediumDate; + import java.time.Year; import java.util.ArrayList; import java.util.Collection; @@ -49,6 +50,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import lombok.extern.slf4j.Slf4j; import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.AnalyticsTableType; import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; @@ -86,7 +88,6 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.util.Assert; -import lombok.extern.slf4j.Slf4j; /** * @author Lars Helge Overland @@ -515,8 +516,8 @@ private List getColumnForDataElement( } /** - * Retyrns a select expression. - * + * Returns a select expression. + * * @param dataElement the {@link DataElement}. * @param columnExpression the column expression. * @return a select expression. @@ -524,7 +525,7 @@ private List getColumnForDataElement( private String getSelectExpression(DataElement dataElement, String columnExpression) { return String.format("%s as %s", columnExpression, quote(dataElement.getUid())); } - + /** * Returns a list of columns. * @@ -536,7 +537,7 @@ private List getColumnForOrgUnitDataElement(DataElement da if (!sqlBuilder.supportsCorrelatedSubquery()) { return List.of(); } - + List columns = new ArrayList<>(); if (isSpatialSupport()) { @@ -618,7 +619,7 @@ private List getColumnForAttributeWithLegendSet( if (!sqlBuilder.supportsCorrelatedSubquery()) { return List.of(); } - + String columnExpression = getColumnExpression(attribute.getValueType(), "value"); String numericClause = getNumericClause("value"); String query = @@ -666,7 +667,7 @@ private List getColumnFromDataElementWithLegendSet( if (!sqlBuilder.supportsCorrelatedSubquery()) { return List.of(); } - + String query = """ (select l.uid from ${maplegend} l \ From 51462efdea0ffcfdcc865a7bb5ef990727754c10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 11 Dec 2024 15:41:08 +0100 Subject: [PATCH 122/170] fix: Update code --- ...dbcTrackedEntityAnalyticsTableManager.java | 38 ++++++++++--------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java index d302e6ee4a0c..0fe6727d8725 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java @@ -177,24 +177,26 @@ private List getColumns( List columns = new ArrayList<>(getFixedColumns()); - String enrolledInProgramExpression = - """ - \s exists(select 1 from ${enrollment} en_0 \ - where en_0.trackedentityid = te.trackedentityid \ - and en_0.programid = ${programId})"""; - - emptyIfNull(programsByTetUid.get(trackedEntityType.getUid())) - .forEach( - program -> - columns.add( - AnalyticsTableColumn.builder() - .name(program.getUid()) - .dataType(BOOLEAN) - .selectExpression( - replaceQualify( - enrolledInProgramExpression, - Map.of("programId", String.valueOf(program.getId())))) - .build())); + if (sqlBuilder.supportsCorrelatedSubquery()) { + String enrolledInProgramExpression = + """ + \s exists(select 1 from ${enrollment} en_0 \ + where en_0.trackedentityid = te.trackedentityid \ + and en_0.programid = ${programId})"""; + + emptyIfNull(programsByTetUid.get(trackedEntityType.getUid())) + .forEach( + program -> + columns.add( + AnalyticsTableColumn.builder() + .name(program.getUid()) + .dataType(BOOLEAN) + .selectExpression( + replaceQualify( + enrolledInProgramExpression, + Map.of("programId", String.valueOf(program.getId())))) + .build())); + } List trackedEntityAttributes = getAllTrackedEntityAttributes(trackedEntityType, programsByTetUid) From 0b838bf80c971c81dfd79d090bb52ecc0de065b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 11 Dec 2024 18:02:26 +0100 Subject: [PATCH 123/170] fix: Update code --- .../analytics/table/JdbcTrackedEntityAnalyticsTableManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java index 8d6fc1f302d6..1a908b1700c2 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java @@ -179,7 +179,7 @@ private List getColumns( List columns = new ArrayList<>(getFixedColumns()); List programs = programsByTetUid.get(trackedEntityType.getUid()); - if (isNotEmpty(programs)) { + if (isNotEmpty(programs) && sqlBuilder.supportsCorrelatedSubquery()) { String enrolledInProgramExpression = """ \s exists(select 1 from ${enrollment} en_0 \ From 96afbba409bf32f7fca64b497d61177d82949afd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 11 Dec 2024 21:13:40 +0100 Subject: [PATCH 124/170] fix: Update code --- .../java/org/hisp/dhis/db/init/AnalyticsDatabaseInit.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/init/AnalyticsDatabaseInit.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/init/AnalyticsDatabaseInit.java index 1bce670ce479..5db921c25c58 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/init/AnalyticsDatabaseInit.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/init/AnalyticsDatabaseInit.java @@ -127,7 +127,9 @@ private void createClickHouseNamedCollection() { "port", config.getIntProperty(ConfigurationKey.CONNECTION_PORT), "database", config.getProperty(ConfigurationKey.CONNECTION_DATABASE), "username", config.getProperty(ConfigurationKey.CONNECTION_USERNAME), - "password", config.getProperty(ConfigurationKey.CONNECTION_PASSWORD)); + "password", config.getProperty(ConfigurationKey.CONNECTION_PASSWORD), + "connection_pool_size", "100", + "replace_query", "1"); ClickHouseSqlBuilder clickHouseSqlBuilder = new ClickHouseSqlBuilder(); From 41bf58efc1a1feafdc8dec9552b47927e769d458 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 11 Dec 2024 21:23:10 +0100 Subject: [PATCH 125/170] fix: Update code --- .../org/hisp/dhis/db/init/AnalyticsDatabaseInit.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/init/AnalyticsDatabaseInit.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/init/AnalyticsDatabaseInit.java index 5db921c25c58..fb0e4f03ac52 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/init/AnalyticsDatabaseInit.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/init/AnalyticsDatabaseInit.java @@ -28,11 +28,8 @@ package org.hisp.dhis.db.init; import static org.hisp.dhis.db.sql.ClickHouseSqlBuilder.NAMED_COLLECTION; - import java.util.Map; import javax.annotation.PostConstruct; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; import org.hisp.dhis.db.SqlBuilderProvider; import org.hisp.dhis.db.model.Database; import org.hisp.dhis.db.setting.SqlBuilderSettings; @@ -43,6 +40,8 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Component; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; /** * Class responsible for performing work for initialization of an analytics database. @@ -127,9 +126,7 @@ private void createClickHouseNamedCollection() { "port", config.getIntProperty(ConfigurationKey.CONNECTION_PORT), "database", config.getProperty(ConfigurationKey.CONNECTION_DATABASE), "username", config.getProperty(ConfigurationKey.CONNECTION_USERNAME), - "password", config.getProperty(ConfigurationKey.CONNECTION_PASSWORD), - "connection_pool_size", "100", - "replace_query", "1"); + "password", config.getProperty(ConfigurationKey.CONNECTION_PASSWORD)); ClickHouseSqlBuilder clickHouseSqlBuilder = new ClickHouseSqlBuilder(); From 0eeac013a46d00ec967dec0b784febdbdccf8ed9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sat, 14 Dec 2024 14:24:29 +0100 Subject: [PATCH 126/170] fix: Update code --- .../DefaultTableReplicationService.java | 60 +++++++++++++++++++ .../TableReplicationService.java | 32 ++++++++++ .../dhis/db/init/AnalyticsDatabaseInit.java | 5 +- 3 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/DefaultTableReplicationService.java create mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/TableReplicationService.java diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/DefaultTableReplicationService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/DefaultTableReplicationService.java new file mode 100644 index 000000000000..d2b48c528c61 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/DefaultTableReplicationService.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.tablereplication; + +import java.util.List; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.hisp.dhis.db.model.Column; +import org.hisp.dhis.db.model.DataType; +import org.hisp.dhis.db.model.Table; +import org.hisp.dhis.db.model.constraint.Nullable; +import org.springframework.stereotype.Service; + +@Slf4j +@Service +@RequiredArgsConstructor +public class DefaultTableReplicationService implements TableReplicationService { + private final TableReplicationStore store; + + @Override + public void replicateTrackedEntityAttributeValue() { + List columns = + List.of( + new Column("trackedentityid", DataType.BIGINT, Nullable.NOT_NULL), + new Column("trackedentityattributeid", DataType.BIGINT, Nullable.NOT_NULL), + new Column("value", DataType.TEXT, Nullable.NULL)); + List primaryKeys = List.of("trackedentityid", "trackedentityattributeid"); + + Table table = new Table("trackedentityattributevalue", columns, primaryKeys); + + store.replicateAnalyticsDatabaseTable(table); + + log.info("Replicated table: '{}'", table.getName()); + } +} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/TableReplicationService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/TableReplicationService.java new file mode 100644 index 000000000000..bdc514c82b7d --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/TableReplicationService.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.tablereplication; + +public interface TableReplicationService { + void replicateTrackedEntityAttributeValue(); +} diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/init/AnalyticsDatabaseInit.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/init/AnalyticsDatabaseInit.java index fb0e4f03ac52..1bce670ce479 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/init/AnalyticsDatabaseInit.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/init/AnalyticsDatabaseInit.java @@ -28,8 +28,11 @@ package org.hisp.dhis.db.init; import static org.hisp.dhis.db.sql.ClickHouseSqlBuilder.NAMED_COLLECTION; + import java.util.Map; import javax.annotation.PostConstruct; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.hisp.dhis.db.SqlBuilderProvider; import org.hisp.dhis.db.model.Database; import org.hisp.dhis.db.setting.SqlBuilderSettings; @@ -40,8 +43,6 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Component; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; /** * Class responsible for performing work for initialization of an analytics database. From a8b798e2c8d5edaeb686e4d839d0b4f1e929a391 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sat, 14 Dec 2024 14:25:47 +0100 Subject: [PATCH 127/170] fix: Update code --- .../DefaultTableReplicationService.java | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/DefaultTableReplicationService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/DefaultTableReplicationService.java index d2b48c528c61..289cee6adb93 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/DefaultTableReplicationService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/DefaultTableReplicationService.java @@ -29,14 +29,12 @@ import java.util.List; import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; import org.hisp.dhis.db.model.Column; import org.hisp.dhis.db.model.DataType; import org.hisp.dhis.db.model.Table; import org.hisp.dhis.db.model.constraint.Nullable; import org.springframework.stereotype.Service; -@Slf4j @Service @RequiredArgsConstructor public class DefaultTableReplicationService implements TableReplicationService { @@ -44,17 +42,15 @@ public class DefaultTableReplicationService implements TableReplicationService { @Override public void replicateTrackedEntityAttributeValue() { - List columns = - List.of( - new Column("trackedentityid", DataType.BIGINT, Nullable.NOT_NULL), - new Column("trackedentityattributeid", DataType.BIGINT, Nullable.NOT_NULL), - new Column("value", DataType.TEXT, Nullable.NULL)); - List primaryKeys = List.of("trackedentityid", "trackedentityattributeid"); - - Table table = new Table("trackedentityattributevalue", columns, primaryKeys); + Table table = + new Table( + "trackedentityattributevalue", + List.of( + new Column("trackedentityid", DataType.BIGINT, Nullable.NOT_NULL), + new Column("trackedentityattributeid", DataType.BIGINT, Nullable.NOT_NULL), + new Column("value", DataType.TEXT, Nullable.NULL)), + List.of("trackedentityid", "trackedentityattributeid")); store.replicateAnalyticsDatabaseTable(table); - - log.info("Replicated table: '{}'", table.getName()); } } From bc075420a5cbc2947cb30a2678f6792cec2f458a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sat, 14 Dec 2024 14:26:37 +0100 Subject: [PATCH 128/170] fix: Update code --- .../DefaultTableReplicationService.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/DefaultTableReplicationService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/DefaultTableReplicationService.java index 289cee6adb93..b03948fa73da 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/DefaultTableReplicationService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/DefaultTableReplicationService.java @@ -38,19 +38,19 @@ @Service @RequiredArgsConstructor public class DefaultTableReplicationService implements TableReplicationService { + private static final Table TABLE_TRACKED_ENTIY_ATTRIBUTE_VALUE = + new Table( + "trackedentityattributevalue", + List.of( + new Column("trackedentityid", DataType.BIGINT, Nullable.NOT_NULL), + new Column("trackedentityattributeid", DataType.BIGINT, Nullable.NOT_NULL), + new Column("value", DataType.TEXT, Nullable.NULL)), + List.of("trackedentityid", "trackedentityattributeid")); + private final TableReplicationStore store; @Override public void replicateTrackedEntityAttributeValue() { - Table table = - new Table( - "trackedentityattributevalue", - List.of( - new Column("trackedentityid", DataType.BIGINT, Nullable.NOT_NULL), - new Column("trackedentityattributeid", DataType.BIGINT, Nullable.NOT_NULL), - new Column("value", DataType.TEXT, Nullable.NULL)), - List.of("trackedentityid", "trackedentityattributeid")); - - store.replicateAnalyticsDatabaseTable(table); + store.replicateAnalyticsDatabaseTable(TABLE_TRACKED_ENTIY_ATTRIBUTE_VALUE); } } From 693bb226d0cf952a8c12251e7787e9ad8d90e9f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sat, 14 Dec 2024 18:47:04 +0100 Subject: [PATCH 129/170] fix: Update code --- .../table/DefaultAnalyticsTableGenerator.java | 15 ++++++++++++++- .../{ => jdbc}/JdbcTableReplicationStore.java | 3 ++- 2 files changed, 16 insertions(+), 2 deletions(-) rename dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/{ => jdbc}/JdbcTableReplicationStore.java (97%) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java index 4abc5d9b0ae5..134cbe84121e 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java @@ -27,6 +27,10 @@ */ package org.hisp.dhis.analytics.table; +import static org.apache.commons.collections4.CollectionUtils.containsAny; +import static org.hisp.dhis.analytics.AnalyticsTableType.ENROLLMENT; +import static org.hisp.dhis.analytics.AnalyticsTableType.EVENT; +import static org.hisp.dhis.analytics.AnalyticsTableType.TRACKED_ENTITY_INSTANCE; import static org.hisp.dhis.common.collection.CollectionUtils.emptyIfNull; import static org.hisp.dhis.scheduling.JobProgress.FailurePolicy.SKIP_STAGE; import static org.hisp.dhis.util.DateUtils.toLongDate; @@ -51,6 +55,7 @@ import org.hisp.dhis.setting.SystemSettings; import org.hisp.dhis.setting.SystemSettingsService; import org.hisp.dhis.system.util.Clock; +import org.hisp.dhis.tablereplication.TableReplicationService; import org.springframework.stereotype.Service; /** @@ -64,6 +69,8 @@ public class DefaultAnalyticsTableGenerator implements AnalyticsTableGenerator { private final ResourceTableService resourceTableService; + private final TableReplicationService tableReplicationService; + private final SystemSettingsService settingsService; private final AnalyticsTableSettings settings; @@ -98,9 +105,15 @@ public void generateAnalyticsTables(AnalyticsTableUpdateParams params0, JobProgr } } + if (!params.isLatestUpdate() && settings.isAnalyticsDatabaseConfigured()) { + if (!containsAny(skipTypes, Set.of(EVENT, ENROLLMENT, TRACKED_ENTITY_INSTANCE))) { + log.info("Replicating tracked entity attribute value table"); + tableReplicationService.replicateTrackedEntityAttributeValue(); + } + } + for (AnalyticsTableService service : analyticsTableServices) { AnalyticsTableType tableType = service.getAnalyticsTableType(); - if (!skipTypes.contains(tableType)) { service.create(params, progress); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/JdbcTableReplicationStore.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/jdbc/JdbcTableReplicationStore.java similarity index 97% rename from dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/JdbcTableReplicationStore.java rename to dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/jdbc/JdbcTableReplicationStore.java index aa5b275c879e..4bddfa16eb33 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/JdbcTableReplicationStore.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/jdbc/JdbcTableReplicationStore.java @@ -25,13 +25,14 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.hisp.dhis.tablereplication; +package org.hisp.dhis.tablereplication.jdbc; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.hisp.dhis.db.model.Table; import org.hisp.dhis.db.sql.SqlBuilder; import org.hisp.dhis.system.util.Clock; +import org.hisp.dhis.tablereplication.TableReplicationStore; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; From 3f30eb9736d188a061aacb633a6f9f007fe6a007 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sat, 14 Dec 2024 20:11:00 +0100 Subject: [PATCH 130/170] fix: Update code --- .../table/JdbcTrackedEntityAnalyticsTableManagerTest.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManagerTest.java index 0d537f2e63da..d70a6e25b7bf 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManagerTest.java @@ -51,7 +51,6 @@ import org.hisp.dhis.program.Program; import org.hisp.dhis.resourcetable.ResourceTableService; import org.hisp.dhis.setting.SystemSettingsProvider; -import org.hisp.dhis.system.util.SqlUtils; import org.hisp.dhis.trackedentity.TrackedEntityAttribute; import org.hisp.dhis.trackedentity.TrackedEntityAttributeService; import org.hisp.dhis.trackedentity.TrackedEntityType; @@ -116,9 +115,6 @@ void verifyNonConfidentialTeasAreSkipped() { Program program = mock(Program.class); - when(sqlBuilder.qualifyTable(anyString())) - .thenAnswer(inv -> SqlUtils.quote(inv.getArgument(0))); - when(sqlBuilder.jsonExtract(anyString(), anyString())).thenReturn("jsonExtract"); when(sqlBuilder.coalesce(anyString(), anyString())) From c193d201b368ca3b02cd410864040afd8b72632f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sat, 14 Dec 2024 20:14:21 +0100 Subject: [PATCH 131/170] fix: Update code --- .../dhis/analytics/table/AbstractEventJdbcTableManager.java | 2 +- .../dhis/analytics/table/JdbcEventAnalyticsTableManager.java | 2 +- .../analytics/table/JdbcTrackedEntityAnalyticsTableManager.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 6af5942d8c5e..508761d83d12 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -287,7 +287,7 @@ private String getOrgUnitSelectSubquery(TrackedEntityAttribute attribute, String protected String getAttributeValueJoinClause(Program program) { String template = """ - left join ${trackedentityattributevalue} as ${uid} \ + left join trackedentityattributevalue as ${uid} \ on en.trackedentityid=${uid}.trackedentityid \ and ${uid}.trackedentityattributeid = ${id}\s"""; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index f5ae60d8af0a..577ca3925f8a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -625,7 +625,7 @@ private List getColumnForAttributeWithLegendSet( String query = """ \s(select l.uid from ${maplegend} l \ - inner join ${trackedentityattributevalue} av on l.startvalue <= ${selectClause} \ + inner join trackedentityattributevalue av on l.startvalue <= ${selectClause} \ and l.endvalue > ${selectClause} \ and l.maplegendsetid=${legendSetId} \ and av.trackedentityid=en.trackedentityid \ diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java index 1a908b1700c2..ceecc38220db 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java @@ -320,7 +320,7 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti sql.append( replaceQualify( """ - \s left join ${trackedentityattributevalue} ${teaUid} on ${teaUid}.trackedentityid=te.trackedentityid \ + \s left join trackedentityattributevalue ${teaUid} on ${teaUid}.trackedentityid=te.trackedentityid \ and ${teaUid}.trackedentityattributeid = ${teaId}""", Map.of( "teaUid", quote(tea.getUid()), From e3ea08efd3c3aa1ea9a43d999b53da6be4e55825 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sat, 14 Dec 2024 20:30:21 +0100 Subject: [PATCH 132/170] fix: Update code --- .../dhis/analytics/table/DefaultAnalyticsTableGenerator.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java index 134cbe84121e..5963aef660d1 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java @@ -27,7 +27,6 @@ */ package org.hisp.dhis.analytics.table; -import static org.apache.commons.collections4.CollectionUtils.containsAny; import static org.hisp.dhis.analytics.AnalyticsTableType.ENROLLMENT; import static org.hisp.dhis.analytics.AnalyticsTableType.EVENT; import static org.hisp.dhis.analytics.AnalyticsTableType.TRACKED_ENTITY_INSTANCE; @@ -91,7 +90,7 @@ public void generateAnalyticsTables(AnalyticsTableUpdateParams params0, JobProgr log.info("Found analytics table types: {}", getAvailableTableTypes()); log.info("Analytics table update params: {}", params); log.info("Last successful analytics table update: {}", toLongDate(lastSuccessfulUpdate)); - log.debug("Skipping table types: {}", skipTypes); + log.info("Skipping table types: {}", skipTypes); progress.startingProcess( "Analytics table update process{}", (params.isLatestUpdate() ? " (latest partition)" : "")); @@ -106,7 +105,7 @@ public void generateAnalyticsTables(AnalyticsTableUpdateParams params0, JobProgr } if (!params.isLatestUpdate() && settings.isAnalyticsDatabaseConfigured()) { - if (!containsAny(skipTypes, Set.of(EVENT, ENROLLMENT, TRACKED_ENTITY_INSTANCE))) { + if (!skipTypes.containsAll(Set.of(EVENT, ENROLLMENT, TRACKED_ENTITY_INSTANCE))) { log.info("Replicating tracked entity attribute value table"); tableReplicationService.replicateTrackedEntityAttributeValue(); } From 4bf8a0a5f42f131e5b1bdb1fb1b279d40a40e263 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sat, 14 Dec 2024 20:31:42 +0100 Subject: [PATCH 133/170] fix: Update code --- .../dhis/analytics/table/DefaultAnalyticsTableGenerator.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java index 5963aef660d1..5f7100195f6c 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java @@ -85,6 +85,7 @@ public void generateAnalyticsTables(AnalyticsTableUpdateParams params0, JobProgr final Date lastSuccessfulUpdate = systemSettings.getLastSuccessfulAnalyticsTablesUpdate(); final AnalyticsTableUpdateParams params = params0.toBuilder().lastSuccessfulUpdate(lastSuccessfulUpdate).build(); + final boolean isAnalyticsDatabase = settings.isAnalyticsDatabaseConfigured(); final Set skipTypes = emptyIfNull(params.getSkipTableTypes()); log.info("Found analytics table types: {}", getAvailableTableTypes()); @@ -98,13 +99,13 @@ public void generateAnalyticsTables(AnalyticsTableUpdateParams params0, JobProgr if (!params.isSkipResourceTables() && !params.isLatestUpdate()) { generateResourceTablesInternal(progress); - if (settings.isAnalyticsDatabaseConfigured()) { + if (isAnalyticsDatabase) { log.info("Replicating resource tables in analytics database"); resourceTableService.replicateAnalyticsResourceTables(); } } - if (!params.isLatestUpdate() && settings.isAnalyticsDatabaseConfigured()) { + if (!params.isLatestUpdate() && isAnalyticsDatabase) { if (!skipTypes.containsAll(Set.of(EVENT, ENROLLMENT, TRACKED_ENTITY_INSTANCE))) { log.info("Replicating tracked entity attribute value table"); tableReplicationService.replicateTrackedEntityAttributeValue(); From 6ffe5d6bda8740c5edf4cd989b9f35fcb29eb640 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sat, 14 Dec 2024 20:34:32 +0100 Subject: [PATCH 134/170] fix: Update code --- .../analytics/table/DefaultAnalyticsTableGenerator.java | 6 +++--- .../analytics/table/setting/AnalyticsTableSettings.java | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java index 5f7100195f6c..039037a5818d 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java @@ -85,12 +85,12 @@ public void generateAnalyticsTables(AnalyticsTableUpdateParams params0, JobProgr final Date lastSuccessfulUpdate = systemSettings.getLastSuccessfulAnalyticsTablesUpdate(); final AnalyticsTableUpdateParams params = params0.toBuilder().lastSuccessfulUpdate(lastSuccessfulUpdate).build(); - final boolean isAnalyticsDatabase = settings.isAnalyticsDatabaseConfigured(); final Set skipTypes = emptyIfNull(params.getSkipTableTypes()); log.info("Found analytics table types: {}", getAvailableTableTypes()); log.info("Analytics table update params: {}", params); log.info("Last successful analytics table update: {}", toLongDate(lastSuccessfulUpdate)); + log.info("Analytics database: %b", settings.isAnalyticsDatabase()); log.info("Skipping table types: {}", skipTypes); progress.startingProcess( @@ -99,13 +99,13 @@ public void generateAnalyticsTables(AnalyticsTableUpdateParams params0, JobProgr if (!params.isSkipResourceTables() && !params.isLatestUpdate()) { generateResourceTablesInternal(progress); - if (isAnalyticsDatabase) { + if (settings.isAnalyticsDatabase()) { log.info("Replicating resource tables in analytics database"); resourceTableService.replicateAnalyticsResourceTables(); } } - if (!params.isLatestUpdate() && isAnalyticsDatabase) { + if (!params.isLatestUpdate() && settings.isAnalyticsDatabase()) { if (!skipTypes.containsAll(Set.of(EVENT, ENROLLMENT, TRACKED_ENTITY_INSTANCE))) { log.info("Replicating tracked entity attribute value table"); tableReplicationService.replicateTrackedEntityAttributeValue(); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/setting/AnalyticsTableSettings.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/setting/AnalyticsTableSettings.java index 645b1aef8154..e77f44a4ca82 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/setting/AnalyticsTableSettings.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/setting/AnalyticsTableSettings.java @@ -111,7 +111,7 @@ public PeriodSource getPeriodSource() { * * @return true if an analytics database instance is configured. */ - public boolean isAnalyticsDatabaseConfigured() { + public boolean isAnalyticsDatabase() { return config.isAnalyticsDatabaseConfigured(); } From a6c1c185eb5b636c13977c8ca41f9d90f984cb64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 15 Dec 2024 11:31:24 +0100 Subject: [PATCH 135/170] fix: Update code --- .../dhis/analytics/table/DefaultAnalyticsTableGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java index 039037a5818d..7bd45ff337b3 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java @@ -90,7 +90,7 @@ public void generateAnalyticsTables(AnalyticsTableUpdateParams params0, JobProgr log.info("Found analytics table types: {}", getAvailableTableTypes()); log.info("Analytics table update params: {}", params); log.info("Last successful analytics table update: {}", toLongDate(lastSuccessfulUpdate)); - log.info("Analytics database: %b", settings.isAnalyticsDatabase()); + log.info("Analytics database: {}", settings.isAnalyticsDatabase()); log.info("Skipping table types: {}", skipTypes); progress.startingProcess( From d5a753dbde3252eaf2c6c19900c71488bc4a8d09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 15 Dec 2024 15:38:56 +0100 Subject: [PATCH 136/170] fix: Update code --- .../hisp/dhis/analytics/table/model/AnalyticsTableColumn.java | 4 ---- .../src/main/java/org/hisp/dhis/db/sql/PostgreSqlBuilder.java | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/model/AnalyticsTableColumn.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/model/AnalyticsTableColumn.java index 537b473493e5..fa49efb4500f 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/model/AnalyticsTableColumn.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/model/AnalyticsTableColumn.java @@ -80,10 +80,6 @@ public class AnalyticsTableColumn { /** Date of creation of the underlying data dimension. */ private final Date created; - // // ------------------------------------------------------------------------- - // // Logic - // // ------------------------------------------------------------------------- - /** Indicates whether this column is not null. */ public boolean isNotNull() { return Nullable.NOT_NULL == nullable; diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgreSqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgreSqlBuilder.java index 92a14270f8cc..df43166bd281 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgreSqlBuilder.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgreSqlBuilder.java @@ -278,6 +278,8 @@ public String dateDifference(String startDate, String endDate, DateUnit dateUnit String.format( "(extract(epoch from (cast(%s as timestamp) - cast(%s as timestamp))) / 60)", endDate, startDate); + case WEEKS -> + String.format("((cast(%s as date) - cast(%s as date)) / 7)", endDate, startDate); case MONTHS -> String.format( "((date_part('year',age(cast(%s as date), cast(%s as date)))) * 12 + " @@ -286,8 +288,6 @@ public String dateDifference(String startDate, String endDate, DateUnit dateUnit case YEARS -> String.format( "(date_part('year',age(cast(%s as date), cast(%s as date))))", endDate, startDate); - case WEEKS -> - String.format("((cast(%s as date) - cast(%s as date)) / 7)", endDate, startDate); }; } From 168796621694c589a0b4bc4aadde60abac1dcdad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 15 Dec 2024 15:39:45 +0100 Subject: [PATCH 137/170] fix: Update code --- .../JdbcTrackedEntityEventsAnalyticsTableManagerTest.java | 4 ++-- .../java/org/hisp/dhis/db/AnalyticsSqlBuilderProvider.java | 4 ++-- ...ticsSqlBuilder.java => PostgreSqlAnalyticsSqlBuilder.java} | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) rename dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/{PostgresAnalyticsSqlBuilder.java => PostgreSqlAnalyticsSqlBuilder.java} (97%) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManagerTest.java index 57fce6c172ac..bc36af519d40 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManagerTest.java @@ -52,8 +52,8 @@ import org.hisp.dhis.category.CategoryService; import org.hisp.dhis.common.IdentifiableObjectManager; import org.hisp.dhis.dataapproval.DataApprovalLevelService; +import org.hisp.dhis.db.sql.PostgreSqlAnalyticsSqlBuilder; import org.hisp.dhis.db.sql.PostgreSqlBuilder; -import org.hisp.dhis.db.sql.PostgresAnalyticsSqlBuilder; import org.hisp.dhis.organisationunit.OrganisationUnitService; import org.hisp.dhis.period.PeriodDataProvider; import org.hisp.dhis.resourcetable.ResourceTableService; @@ -82,7 +82,7 @@ class JdbcTrackedEntityEventsAnalyticsTableManagerTest { @Spy private PostgreSqlBuilder sqlBuilder; - @Spy private PostgresAnalyticsSqlBuilder analyticsSqlBuilder; + @Spy private PostgreSqlAnalyticsSqlBuilder analyticsSqlBuilder; @Mock private PartitionManager partitionManager; diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/AnalyticsSqlBuilderProvider.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/AnalyticsSqlBuilderProvider.java index 9f7aac27027c..3c6728876f0d 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/AnalyticsSqlBuilderProvider.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/AnalyticsSqlBuilderProvider.java @@ -33,7 +33,7 @@ import org.hisp.dhis.db.sql.AnalyticsSqlBuilder; import org.hisp.dhis.db.sql.ClickhouseAnalyticsSqlBuilder; import org.hisp.dhis.db.sql.DorisAnalyticsSqlBuilder; -import org.hisp.dhis.db.sql.PostgresAnalyticsSqlBuilder; +import org.hisp.dhis.db.sql.PostgreSqlAnalyticsSqlBuilder; import org.hisp.dhis.external.conf.DhisConfigurationProvider; import org.springframework.stereotype.Service; @@ -70,7 +70,7 @@ private AnalyticsSqlBuilder getSqlBuilder(SqlBuilderSettings config) { return switch (database) { case DORIS -> new DorisAnalyticsSqlBuilder(); case CLICKHOUSE -> new ClickhouseAnalyticsSqlBuilder(); - default -> new PostgresAnalyticsSqlBuilder(); + default -> new PostgreSqlAnalyticsSqlBuilder(); }; } } diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgresAnalyticsSqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgreSqlAnalyticsSqlBuilder.java similarity index 97% rename from dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgresAnalyticsSqlBuilder.java rename to dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgreSqlAnalyticsSqlBuilder.java index 1e1e5dd5fbea..115b7b022c30 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgresAnalyticsSqlBuilder.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgreSqlAnalyticsSqlBuilder.java @@ -27,7 +27,7 @@ */ package org.hisp.dhis.db.sql; -public class PostgresAnalyticsSqlBuilder implements AnalyticsSqlBuilder { +public class PostgreSqlAnalyticsSqlBuilder implements AnalyticsSqlBuilder { /** * Returns a subquery that expand the event datavalue jsonb with two additional fields: From 744704f7f0a943b02a22b793dab15e7a4b036c50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 15 Dec 2024 15:53:44 +0100 Subject: [PATCH 138/170] fix: Update code --- .../AbstractJdbcEventAnalyticsManager.java | 18 ++++------- .../query/context/sql/SqlQueryBuilders.java | 32 +++++++++---------- .../dhis/analytics/util/DisplayNameUtils.java | 12 +++---- 3 files changed, 27 insertions(+), 35 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java index 9f53bfd51add..55a58883bd63 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java @@ -49,8 +49,8 @@ import static org.hisp.dhis.analytics.event.data.EnrollmentQueryHelper.getHeaderColumns; import static org.hisp.dhis.analytics.event.data.EnrollmentQueryHelper.getOrgUnitLevelColumns; import static org.hisp.dhis.analytics.event.data.EnrollmentQueryHelper.getPeriodColumns; -import static org.hisp.dhis.analytics.table.JdbcEventAnalyticsTableManager.OU_GEOMETRY_COL_SUFFIX; -import static org.hisp.dhis.analytics.table.JdbcEventAnalyticsTableManager.OU_NAME_COL_SUFFIX; +import static org.hisp.dhis.analytics.table.AbstractEventJdbcTableManager.OU_GEOMETRY_COL_SUFFIX; +import static org.hisp.dhis.analytics.table.AbstractEventJdbcTableManager.OU_NAME_COL_SUFFIX; import static org.hisp.dhis.analytics.util.AnalyticsUtils.replaceStringBetween; import static org.hisp.dhis.analytics.util.AnalyticsUtils.throwIllegalQueryEx; import static org.hisp.dhis.analytics.util.AnalyticsUtils.withExceptionHandling; @@ -375,8 +375,9 @@ private void addDimensionSelectColumns( singleQuote(period.getIsoDate()) + " as " + period.getPeriodType().getName()); } else { throw new IllegalStateException( - "Program indicator non-default boundary query must have " - + "exactly one period, or no periods and a period filter"); + """ + Program indicator non-default boundary query must have \" + exactly one period, or no periods and a period filter"""); } }); } @@ -455,7 +456,6 @@ private ColumnAndAlias getColumnAndAlias( } else if (queryItem.getValueType() == ValueType.NUMBER && !isGroupByClause) { ColumnAndAlias columnAndAlias = getColumnAndAlias(queryItem, isAggregated, queryItem.getItemName()); - return ColumnAndAlias.ofColumnAndAlias( columnAndAlias.getColumn(), defaultIfNull(columnAndAlias.getAlias(), queryItem.getItemName())); @@ -532,14 +532,10 @@ protected Optional getAlias(QueryItem queryItem) { @Transactional(readOnly = true, propagation = REQUIRES_NEW) public Grid getAggregatedEventData(EventQueryParams params, Grid grid, int maxLimit) { String aggregateClause = getAggregateClause(params); + String columns = StringUtils.join(getSelectColumns(params, true), ","); String sql = - TextUtils.removeLastComma( - "select " - + aggregateClause - + " as value," - + StringUtils.join(getSelectColumns(params, true), ",") - + " "); + TextUtils.removeLastComma("select " + aggregateClause + " as value," + columns + " "); // --------------------------------------------------------------------- // Criteria diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/trackedentity/query/context/sql/SqlQueryBuilders.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/trackedentity/query/context/sql/SqlQueryBuilders.java index e9768189d49c..23ae04e3abd6 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/trackedentity/query/context/sql/SqlQueryBuilders.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/trackedentity/query/context/sql/SqlQueryBuilders.java @@ -61,22 +61,22 @@ select json_agg(json_build_object( // incidentDate is kept to support a deprecated field, will be removed when FE will only // use occurreddate """ - select json_agg( - json_build_object( - 'programUid', en.program, - 'enrollmentUid', en.enrollment, - 'enrollmentDate', en.enrollmentdate, - 'incidentDate', en.occurreddate, - 'occurredDate', en.occurreddate, - 'completedDate', en.completeddate, - 'orgUnitUid', en.ou, - 'orgUnitName', en.ouname, - 'orgUnitCode', en.oucode, - 'orgUnitNameHierarchy', en.ounamehierarchy, - 'enrollmentStatus', en.enrollmentstatus, - 'events', ${eventQuery})) - from analytics_te_enrollment_${trackedEntityType} en - where en.trackedentity = t_1.trackedentity""", + select json_agg( + json_build_object( + 'programUid', en.program, + 'enrollmentUid', en.enrollment, + 'enrollmentDate', en.enrollmentdate, + 'incidentDate', en.occurreddate, + 'occurredDate', en.occurreddate, + 'completedDate', en.completeddate, + 'orgUnitUid', en.ou, + 'orgUnitName', en.ouname, + 'orgUnitCode', en.oucode, + 'orgUnitNameHierarchy', en.ounamehierarchy, + 'enrollmentStatus', en.enrollmentstatus, + 'events', ${eventQuery})) + from analytics_te_enrollment_${trackedEntityType} en + where en.trackedentity = t_1.trackedentity""", Map.of("eventQuery", coalesceToEmptyArray(EVENT_QUERY))); private static final String JSON_AGGREGATION_QUERY = coalesceToEmptyArray(ENROLLMENT_QUERY); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/DisplayNameUtils.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/DisplayNameUtils.java index 5377f0807253..b7f47fbce85d 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/DisplayNameUtils.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/DisplayNameUtils.java @@ -124,22 +124,22 @@ public static String getDisplayName( isNotEmpty.apply(surname), isNotEmpty.apply(firstName), isEmpty.apply(username), - formatNames(sqlBuilder, surname, "', '", firstName), + sqlBuilder.concat(surname, "', '", firstName), // FirstName and Username isEmpty.apply(surname), isNotEmpty.apply(firstName), isNotEmpty.apply(username), - formatNames(sqlBuilder, firstName, "' ('", username, "')'"), + sqlBuilder.concat(firstName, "' ('", username, "')'"), // Surname and Username isNotEmpty.apply(surname), isEmpty.apply(firstName), isNotEmpty.apply(username), - formatNames(sqlBuilder, surname, "' ('", username, "')'"), + sqlBuilder.concat(surname, "' ('", username, "')'"), // All fields - formatNames(sqlBuilder, surname, "', '", firstName, "' ('", username, "')'"), + sqlBuilder.concat(surname, "', '", firstName, "' ('", username, "')'"), columnAlias); } @@ -149,8 +149,4 @@ private static String extractJsonValue( String jsonExtracted = sqlBuilder.jsonExtract(json, path); return sqlBuilder.trim(jsonExtracted); } - - private static String formatNames(SqlBuilder sqlBuilder, String... elements) { - return sqlBuilder.concat(elements); - } } From f70715bb2c6b9606f66afd68f8bb9963bd039fa4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 15 Dec 2024 16:07:24 +0100 Subject: [PATCH 139/170] fix: Update code --- .../java/org/hisp/dhis/analytics/util/DisplayNameUtils.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/DisplayNameUtils.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/DisplayNameUtils.java index b7f47fbce85d..fd07c00ff505 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/DisplayNameUtils.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/DisplayNameUtils.java @@ -28,9 +28,9 @@ package org.hisp.dhis.analytics.util; import java.util.function.UnaryOperator; +import org.hisp.dhis.db.sql.SqlBuilder; import lombok.AccessLevel; import lombok.NoArgsConstructor; -import org.hisp.dhis.db.sql.SqlBuilder; @NoArgsConstructor(access = AccessLevel.PRIVATE) public final class DisplayNameUtils { @@ -64,6 +64,8 @@ public static String getDisplayName( String surname = extractJsonValue(sqlBuilder, tablePrefix, originColumn, "surname"); String firstName = extractJsonValue(sqlBuilder, tablePrefix, originColumn, "firstName"); String username = extractJsonValue(sqlBuilder, tablePrefix, originColumn, "username"); + + // return sqlBuilder.concat(surname, "', '", firstName, "' ('", username, "')'"); // Helper methods for the CASE conditions UnaryOperator isEmpty = expression -> sqlBuilder.coalesce(expression, "''") + " = ''"; From 721f1d247a27c13cda46ba5f1a73a419c1029745 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 15 Dec 2024 16:08:23 +0100 Subject: [PATCH 140/170] fix: Update code --- .../dhis/analytics/util/DisplayNameUtils.java | 101 +----------------- 1 file changed, 3 insertions(+), 98 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/DisplayNameUtils.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/DisplayNameUtils.java index fd07c00ff505..2ce31672bb39 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/DisplayNameUtils.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/DisplayNameUtils.java @@ -27,32 +27,14 @@ */ package org.hisp.dhis.analytics.util; -import java.util.function.UnaryOperator; -import org.hisp.dhis.db.sql.SqlBuilder; import lombok.AccessLevel; import lombok.NoArgsConstructor; +import org.hisp.dhis.db.sql.SqlBuilder; @NoArgsConstructor(access = AccessLevel.PRIVATE) public final class DisplayNameUtils { /** - * TODO Refactor and change this, too much code vs benefit. - * - *

This method will extract/compose the display name, based on the tracker JSON objects living - * in the 'originColumn'. This method will return the display name respecting these rules: - * - *

If (last name, first name and username) are populated => Last name, first name (username) - * - *

If (only username is populated) => username - * - *

If (only first name is populated) => first name - * - *

If (only last name is populated) => last name - * - *

If (only last name and first name are populated) => last name, first name - * - *

If (only last name and username are populated) => last name (username) - * - *

If (only first name and username are populated) => first name (username) + * Creates a display name from a user info JSON object. * * @param originColumn the original column from where the JSON values are extracted from * @param tablePrefix the prefix of the tracker table @@ -64,85 +46,8 @@ public static String getDisplayName( String surname = extractJsonValue(sqlBuilder, tablePrefix, originColumn, "surname"); String firstName = extractJsonValue(sqlBuilder, tablePrefix, originColumn, "firstName"); String username = extractJsonValue(sqlBuilder, tablePrefix, originColumn, "username"); - - // return sqlBuilder.concat(surname, "', '", firstName, "' ('", username, "')'"); - - // Helper methods for the CASE conditions - UnaryOperator isEmpty = expression -> sqlBuilder.coalesce(expression, "''") + " = ''"; - - UnaryOperator isNotEmpty = - expression -> sqlBuilder.coalesce(expression, "''") + " <> ''"; - - return String.format( - "case" - + - // All empty - " when %s and %s and %s then null" - + - // Username only - " when %s and %s and %s then %s" - + - // FirstName only - " when %s and %s and %s then %s" - + - // Surname only - " when %s and %s and %s then %s" - + - // Surname and FirstName - " when %s and %s and %s then %s" - + - // FirstName and Username - " when %s and %s and %s then %s" - + - // Surname and Username - " when %s and %s and %s then %s" - + - // All fields - " else %s end as %s", - // All empty - isEmpty.apply(surname), - isEmpty.apply(firstName), - isEmpty.apply(username), - - // Username only - isEmpty.apply(surname), - isEmpty.apply(firstName), - isNotEmpty.apply(username), - username, - - // FirstName only - isEmpty.apply(surname), - isNotEmpty.apply(firstName), - isEmpty.apply(username), - firstName, - - // Surname only - isNotEmpty.apply(surname), - isEmpty.apply(firstName), - isEmpty.apply(username), - surname, - - // Surname and FirstName - isNotEmpty.apply(surname), - isNotEmpty.apply(firstName), - isEmpty.apply(username), - sqlBuilder.concat(surname, "', '", firstName), - - // FirstName and Username - isEmpty.apply(surname), - isNotEmpty.apply(firstName), - isNotEmpty.apply(username), - sqlBuilder.concat(firstName, "' ('", username, "')'"), - - // Surname and Username - isNotEmpty.apply(surname), - isEmpty.apply(firstName), - isNotEmpty.apply(username), - sqlBuilder.concat(surname, "' ('", username, "')'"), - // All fields - sqlBuilder.concat(surname, "', '", firstName, "' ('", username, "')'"), - columnAlias); + return sqlBuilder.concat(surname, "', '", firstName, "' ('", username, "')'"); } private static String extractJsonValue( From a879c8bdda48f24fd47d8ac18e77cc808133c650 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 15 Dec 2024 17:53:38 +0100 Subject: [PATCH 141/170] fix: Update code --- .../JdbcTrackedEntityAnalyticsTableManagerTest.java | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManagerTest.java index d70a6e25b7bf..7e596afc3000 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManagerTest.java @@ -30,7 +30,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -45,6 +44,7 @@ import org.hisp.dhis.common.ValueType; import org.hisp.dhis.dataapproval.DataApprovalLevelService; import org.hisp.dhis.db.model.Column; +import org.hisp.dhis.db.sql.PostgreSqlBuilder; import org.hisp.dhis.db.sql.SqlBuilder; import org.hisp.dhis.organisationunit.OrganisationUnitService; import org.hisp.dhis.period.PeriodDataProvider; @@ -59,6 +59,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.Spy; import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.jdbc.core.JdbcTemplate; @@ -70,7 +71,7 @@ class JdbcTrackedEntityAnalyticsTableManagerTest { @Mock private PeriodDataProvider periodDataProvider; - @Mock private SqlBuilder sqlBuilder; + @Spy private SqlBuilder sqlBuilder = new PostgreSqlBuilder(); @Mock private PartitionManager partitionManager; @@ -115,13 +116,6 @@ void verifyNonConfidentialTeasAreSkipped() { Program program = mock(Program.class); - when(sqlBuilder.jsonExtract(anyString(), anyString())).thenReturn("jsonExtract"); - - when(sqlBuilder.coalesce(anyString(), anyString())) - .thenAnswer(inv -> "coalesce(" + inv.getArgument(0) + ", " + inv.getArgument(1) + "))"); - - when(sqlBuilder.trim(anyString())).thenAnswer(inv -> "trim(" + inv.getArgument(0) + ")"); - when(tet.getTrackedEntityAttributes()).thenReturn(List.of(nonConfidentialTea, confidentialTea)); when(program.getTrackedEntityType()).thenReturn(tet); From e727895de4abe420946176b7872b220b6b75983f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 15 Dec 2024 17:58:20 +0100 Subject: [PATCH 142/170] fix: Update code --- .../java/org/hisp/dhis/analytics/util/DisplayNameUtils.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/DisplayNameUtils.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/DisplayNameUtils.java index 2ce31672bb39..29c7b0e1b57f 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/DisplayNameUtils.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/DisplayNameUtils.java @@ -46,8 +46,9 @@ public static String getDisplayName( String surname = extractJsonValue(sqlBuilder, tablePrefix, originColumn, "surname"); String firstName = extractJsonValue(sqlBuilder, tablePrefix, originColumn, "firstName"); String username = extractJsonValue(sqlBuilder, tablePrefix, originColumn, "username"); + String expression = sqlBuilder.concat(surname, "', '", firstName, "' ('", username, "')'"); - return sqlBuilder.concat(surname, "', '", firstName, "' ('", username, "')'"); + return String.format("%s as %s", expression, columnAlias); } private static String extractJsonValue( From b0986d3f3301168b6b5a355b8f91c17871bff1f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 16 Dec 2024 10:26:13 +0100 Subject: [PATCH 143/170] fix: Update code --- .../table/AbstractJdbcTableManager.java | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java index c7c63da03aae..d37c773f20de 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java @@ -230,19 +230,18 @@ public void swapTable(AnalyticsTableUpdateParams params, AnalyticsTable table) { boolean skipMasterTable = params.isPartialUpdate() && tableExists && table.getTableType().isLatestPartition(); - log.info( - "Swapping table, master table exists: '{}', skip master table: '{}'", - tableExists, - skipMasterTable); - List swappedPartitions = new UniqueArrayList<>(); + log.info("Swapping table: '%s'", table.getMainName()); + log.info("Master table exists: '{}', skip master table: '{}'", tableExists, skipMasterTable); + + List
swapPartitions = new UniqueArrayList<>(); table.getTablePartitions().stream() - .forEach(p -> swappedPartitions.add(swapTable(p, p.getMainName()))); + .forEach(p -> swapPartitions.add(swapTable(p, p.getMainName()))); if (!skipMasterTable) { swapTable(table, table.getMainName()); } else { - swappedPartitions.forEach( - partition -> swapInheritance(partition, table.getName(), table.getMainName())); + swapPartitions.forEach( + partition -> swapParentTable(partition, table.getName(), table.getMainName())); dropTable(table); } } @@ -301,7 +300,7 @@ private Table swapTable(Table stagingTable, String mainTableName) { * @param stagingMasterName the staging master table name. * @param mainMasterName the main master table name. */ - private void swapInheritance(Table partition, String stagingMasterName, String mainMasterName) { + private void swapParentTable(Table partition, String stagingMasterName, String mainMasterName) { executeSilently(sqlBuilder.swapParentTable(partition, stagingMasterName, mainMasterName)); } From 115fddc0152a9cc2020e5acd7a3567def65e4fab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 16 Dec 2024 10:33:09 +0100 Subject: [PATCH 144/170] fix: Update code --- .../analytics/table/AbstractJdbcTableManager.java | 14 ++++++++++++-- .../org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java | 5 +++++ .../java/org/hisp/dhis/db/sql/DorisSqlBuilder.java | 5 +++++ .../org/hisp/dhis/db/sql/PostgreSqlBuilder.java | 5 +++++ .../main/java/org/hisp/dhis/db/sql/SqlBuilder.java | 7 ++++++- 5 files changed, 33 insertions(+), 3 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java index d37c773f20de..d1ab33895562 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java @@ -289,7 +289,12 @@ public int invokeAnalyticsTableSqlHooks() { * @param mainTableName the main table name. */ private Table swapTable(Table stagingTable, String mainTableName) { - executeSilently(sqlBuilder.swapTable(stagingTable, mainTableName)); + if (sqlBuilder.supportsMultiStatements()) { + executeSilently(sqlBuilder.swapTable(stagingTable, mainTableName)); + } else { + executeSilently(sqlBuilder.dropTableIfExistsCascade(mainTableName)); + executeSilently(sqlBuilder.renameTable(stagingTable, mainTableName)); + } return stagingTable.swapFromStaging(); } @@ -301,7 +306,12 @@ private Table swapTable(Table stagingTable, String mainTableName) { * @param mainMasterName the main master table name. */ private void swapParentTable(Table partition, String stagingMasterName, String mainMasterName) { - executeSilently(sqlBuilder.swapParentTable(partition, stagingMasterName, mainMasterName)); + if (sqlBuilder.supportsMultiStatements()) { + executeSilently(sqlBuilder.swapParentTable(partition, stagingMasterName, mainMasterName)); + } else { + sqlBuilder.removeParentTable(partition, stagingMasterName); + sqlBuilder.setParentTable(partition, mainMasterName); + } } /** diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java index 6381be607075..3e0b5296d96a 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java @@ -169,6 +169,11 @@ public boolean supportsCorrelatedSubquery() { return false; } + @Override + public boolean supportsMultiStatements() { + return false; + } + @Override public boolean requiresIndexesForAnalytics() { return false; diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java index 48bfe6bc31b8..05fb138a7138 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java @@ -171,6 +171,11 @@ public boolean supportsCorrelatedSubquery() { return true; } + @Override + public boolean supportsMultiStatements() { + return true; + } + @Override public boolean requiresIndexesForAnalytics() { return false; diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgreSqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgreSqlBuilder.java index df43166bd281..1649e0e50fd1 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgreSqlBuilder.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgreSqlBuilder.java @@ -190,6 +190,11 @@ public boolean supportsCorrelatedSubquery() { return true; } + @Override + public boolean supportsMultiStatements() { + return true; + } + @Override public boolean requiresIndexesForAnalytics() { return true; diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java index 15f167775fc4..205dc2fb238f 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java @@ -175,10 +175,15 @@ public interface SqlBuilder { boolean supportsVacuum(); /** - * @return true if the DBMS supports corrected subqueries. + * @return true if the DBMS supports correlated subqueries. */ boolean supportsCorrelatedSubquery(); + /** + * @return true if the DMBS supports multiple statements in one operation. + */ + boolean supportsMultiStatements(); + /** * @return true if the DBMS requires indexes for analytics tables for performance. */ From 604469b20d626a3b6460801a993e846bc8ad603a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 16 Dec 2024 11:07:00 +0100 Subject: [PATCH 145/170] fix: Update code --- .../hisp/dhis/analytics/table/AbstractJdbcTableManager.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java index d1ab33895562..fc4865bb9088 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java @@ -230,8 +230,9 @@ public void swapTable(AnalyticsTableUpdateParams params, AnalyticsTable table) { boolean skipMasterTable = params.isPartialUpdate() && tableExists && table.getTableType().isLatestPartition(); - log.info("Swapping table: '%s'", table.getMainName()); - log.info("Master table exists: '{}', skip master table: '{}'", tableExists, skipMasterTable); + log.info("Swapping table: '{}'", table.getMainName()); + log.info("Master table exists: {}, skip master table: {}", tableExists, skipMasterTable); + log.info("Supports multi-statements: {}", sqlBuilder.supportsMultiStatements()); List
swapPartitions = new UniqueArrayList<>(); table.getTablePartitions().stream() From 791ec72144cf653e122cf1afc123cb3245353af5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 16 Dec 2024 12:32:06 +0100 Subject: [PATCH 146/170] fix: Update code --- .../hisp/dhis/analytics/table/AbstractJdbcTableManager.java | 4 ++-- .../dhis/analytics/table/JdbcAnalyticsTableManagerTest.java | 2 +- .../src/main/java/org/hisp/dhis/db/model/Table.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java index fc4865bb9088..00adb7d802f1 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java @@ -236,7 +236,7 @@ public void swapTable(AnalyticsTableUpdateParams params, AnalyticsTable table) { List
swapPartitions = new UniqueArrayList<>(); table.getTablePartitions().stream() - .forEach(p -> swapPartitions.add(swapTable(p, p.getMainName()))); + .forEach(partition -> swapPartitions.add(swapTable(partition, partition.getMainName()))); if (!skipMasterTable) { swapTable(table, table.getMainName()); @@ -296,7 +296,7 @@ private Table swapTable(Table stagingTable, String mainTableName) { executeSilently(sqlBuilder.dropTableIfExistsCascade(mainTableName)); executeSilently(sqlBuilder.renameTable(stagingTable, mainTableName)); } - return stagingTable.swapFromStaging(); + return stagingTable.fromStaging(); } /** diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerTest.java index 7c968b08511a..85acb0f9ce4c 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerTest.java @@ -314,7 +314,7 @@ void testSwapTable() { when(jdbcTemplate.queryForList(any())).thenReturn(List.of(Map.of("table_name", "analytic"))); - Table swappedPartition = table.getTablePartitions().get(0).swapFromStaging(); + Table swappedPartition = table.getTablePartitions().get(0).fromStaging(); subject.swapTable(params, table); assertEquals("analytics_2023_temp", table.getTablePartitions().get(0).getName()); assertEquals("analytics_2023", swappedPartition.getName()); diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/model/Table.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/model/Table.java index 2c0c83f85d04..a1db5bb0f540 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/model/Table.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/model/Table.java @@ -254,7 +254,7 @@ public boolean hasPartitions() { return isNotEmpty(partitions); } - public Table swapFromStaging() { + public Table fromStaging() { return new Table( fromStaging(this.getName()), this.getColumns(), From ecec55604cc87dc9dd2a024e7b2339b2a3deb0b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 16 Dec 2024 13:30:17 +0100 Subject: [PATCH 147/170] fix: Update code --- .../analytics/table/AbstractJdbcTableManager.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java index 37347243859c..188320fac198 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java @@ -292,7 +292,12 @@ public int invokeAnalyticsTableSqlHooks() { * @param mainTableName the main table name. */ private void swapTable(Table stagingTable, String mainTableName) { - executeSilently(sqlBuilder.swapTable(stagingTable, mainTableName)); + if (sqlBuilder.supportsMultiStatements()) { + executeSilently(sqlBuilder.swapTable(stagingTable, mainTableName)); + } else { + executeSilently(sqlBuilder.dropTableIfExistsCascade(mainTableName)); + executeSilently(sqlBuilder.renameTable(stagingTable, mainTableName)); + } } /** @@ -303,7 +308,12 @@ private void swapTable(Table stagingTable, String mainTableName) { * @param mainMasterName the main master table name. */ private void swapParentTable(Table partition, String stagingMasterName, String mainMasterName) { - executeSilently(sqlBuilder.swapParentTable(partition, stagingMasterName, mainMasterName)); + if (sqlBuilder.supportsMultiStatements()) { + executeSilently(sqlBuilder.swapParentTable(partition, stagingMasterName, mainMasterName)); + } else { + executeSilently(sqlBuilder.removeParentTable(partition, stagingMasterName)); + executeSilently(sqlBuilder.setParentTable(partition, mainMasterName)); + } } /** From def2c082faa98b4c551548cc8e87efaed13af32c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 16 Dec 2024 13:31:11 +0100 Subject: [PATCH 148/170] fix: Update code --- .../table/DefaultAnalyticsTableService.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java index 08e396d4ceea..66a471b3f26e 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java @@ -88,14 +88,13 @@ public void create(AnalyticsTableUpdateParams params, JobProgress progress) { AnalyticsTableType tableType = getAnalyticsTableType(); - Clock clock = - new Clock(log) - .startClock() - .logTime( - "Starting update of type: {}, table name: '{}', parallel jobs: {}", - tableType, - tableType.getTableName(), - parallelJobs); + Clock clock = new Clock(log).startClock(); + + clock.logTime( + "Starting update of type: {}, table name: '{}', parallel jobs: {}", + tableType, + tableType.getTableName(), + parallelJobs); progress.startingStage("Validating analytics table: {}", tableType); boolean validState = tableManager.validState(); From b2a7701c05e4d2c8205ad846558ec6ab8e4dd32a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 16 Dec 2024 13:31:45 +0100 Subject: [PATCH 149/170] fix: Update code --- .../hisp/dhis/analytics/table/DefaultAnalyticsTableService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java index 66a471b3f26e..19b029cc2963 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java @@ -89,7 +89,6 @@ public void create(AnalyticsTableUpdateParams params, JobProgress progress) { AnalyticsTableType tableType = getAnalyticsTableType(); Clock clock = new Clock(log).startClock(); - clock.logTime( "Starting update of type: {}, table name: '{}', parallel jobs: {}", tableType, From 0c00de71250d86cb9002c0646dd20cb59fee4b2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 16 Dec 2024 13:33:25 +0100 Subject: [PATCH 150/170] fix: Update code --- .../analytics/table/AbstractJdbcTableManager.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java index 188320fac198..d59263be6dad 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java @@ -235,16 +235,20 @@ public void swapTable(AnalyticsTableUpdateParams params, AnalyticsTable table) { List
swappedPartitions = new UniqueArrayList<>(); - table.getTablePartitions().forEach(part -> swapTable(part, part.getMainName())); - table.getTablePartitions().forEach(part -> swappedPartitions.add(part.fromStaging())); + if (!sqlBuilder.supportsDeclarativePartitioning()) { + table.getTablePartitions().forEach(part -> swapTable(part, part.getMainName())); + table.getTablePartitions().forEach(part -> swappedPartitions.add(part.fromStaging())); + } if (!skipMasterTable) { // Full replace update and main table exist, swap main table swapTable(table, table.getMainName()); } else { // Incremental append update, update parent of partitions to existing main table - swappedPartitions.forEach( - partition -> swapParentTable(partition, table.getName(), table.getMainName())); + if (!sqlBuilder.supportsDeclarativePartitioning()) { + swappedPartitions.forEach( + partition -> swapParentTable(partition, table.getName(), table.getMainName())); + } dropTable(table); } } From bb3bf3e8315d9d5fde62961c10d37d59a1da6f28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 16 Dec 2024 14:09:03 +0100 Subject: [PATCH 151/170] fix: Update code --- .../table/DefaultAnalyticsTableService.java | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java index 19b029cc2963..654351d07d4d 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java @@ -28,6 +28,7 @@ package org.hisp.dhis.analytics.table; import static org.hisp.dhis.analytics.util.AnalyticsIndexHelper.getIndexes; +import static org.hisp.dhis.commons.util.TextUtils.format; import static org.hisp.dhis.scheduling.JobProgress.FailurePolicy.SKIP_ITEM_OUTLIER; import static org.hisp.dhis.scheduling.JobProgress.FailurePolicy.SKIP_STAGE; import static org.hisp.dhis.util.DateUtils.toLongDate; @@ -107,7 +108,7 @@ public void create(AnalyticsTableUpdateParams params, JobProgress progress) { if (tables.isEmpty()) { clock.logTime("Table update aborted, nothing to update: '{}'", tableType.getTableName()); - progress.startingStage("Table updates " + tableType); + progress.startingStage("Table update of type: '{}'", tableType); progress.completedStage("Table updated aborted, no table or partitions to be updated"); return; } @@ -119,13 +120,13 @@ public void create(AnalyticsTableUpdateParams params, JobProgress progress) { params); progress.startingStage("Performing pre-create table work"); progress.runStage(() -> tableManager.preCreateTables(params)); - clock.logTime("Performed pre-create table work " + tableType); + clock.logTime("Performed pre-create table work: '{}'", tableType); - progress.startingStage("Dropping staging tables (if any) " + tableType, tables.size()); + progress.startingStage(format("Dropping staging tables: '{}'", tableType), tables.size()); dropTables(tables, progress); clock.logTime("Dropped staging tables"); - progress.startingStage("Creating analytics tables " + tableType, tables.size()); + progress.startingStage(format("Creating analytics tables: '{}'", tableType), tables.size()); createTables(tables, progress); clock.logTime("Created analytics tables"); @@ -133,11 +134,11 @@ public void create(AnalyticsTableUpdateParams params, JobProgress progress) { int partitionSize = partitions.size(); progress.startingStage( - "Populating " + partitionSize + " analytics tables " + tableType, partitionSize); + format("Populating {} analytics tables: '{}'", partitionSize, tableType), partitionSize); populateTables(params, partitions, progress); clock.logTime("Populated analytics tables"); - progress.startingStage("Invoking analytics table hooks " + tableType); + progress.startingStage("Invoking analytics table hooks: '{}'", tableType); tableUpdates += progress.runStage(0, tableManager::invokeAnalyticsTableSqlHooks); clock.logTime("Invoked analytics table hooks"); @@ -148,25 +149,27 @@ public void create(AnalyticsTableUpdateParams params, JobProgress progress) { List indexes = getIndexes(partitions); int indexSize = indexes.size(); progress.startingStage( - "Creating " + indexSize + " indexes " + tableType, indexSize, SKIP_ITEM_OUTLIER); + format("Creating {} indexes: '{}'", indexSize, tableType), indexSize, SKIP_ITEM_OUTLIER); createIndexes(indexes, progress); clock.logTime("Created indexes"); } if (tableUpdates > 0 && sqlBuilder.supportsVacuum()) { - progress.startingStage("Vacuuming tables " + tableType, partitions.size()); + progress.startingStage(format("Vacuuming tables: '{}'", tableType), partitions.size()); vacuumTables(partitions, progress); clock.logTime("Tables vacuumed"); } if (sqlBuilder.supportsAnalyze()) { - progress.startingStage("Analyzing analytics tables " + tableType, partitions.size()); + progress.startingStage( + format("Analyzing analytics tables: '{}'", tableType), partitions.size()); analyzeTables(partitions, progress); clock.logTime("Analyzed tables"); } if (params.isLatestUpdate()) { - progress.startingStage("Removing updated and deleted data " + tableType, SKIP_STAGE); + progress.startingStage( + format("Removing updated and deleted data: '{}'", tableType), SKIP_STAGE); progress.runStage(() -> tableManager.removeUpdatedData(tables)); clock.logTime("Removed updated and deleted data"); } @@ -263,7 +266,7 @@ private int applyAggregationLevels( if (!dataElements.isEmpty()) { progress.startingStage( - "Applying aggregation level " + level + " " + tableType, tables.size()); + format("Applying aggregation level {}: '{}'", level, tableType), tables.size()); progress.runStageInParallel( getParallelJobs(), tables, @@ -321,7 +324,8 @@ private void swapTables( AnalyticsTableUpdateParams params, List tables, JobProgress progress) { resourceTableService.dropAllSqlViews(progress); - progress.startingStage("Swapping analytics tables " + getAnalyticsTableType(), tables.size()); + progress.startingStage( + format("Swapping analytics tables: '{}'", getAnalyticsTableType()), tables.size()); progress.runStage( tables, AnalyticsTable::getName, table -> tableManager.swapTable(params, table)); From bdaf60fe769dc83a953b2ee83ea09183aecdfac4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 16 Dec 2024 14:12:07 +0100 Subject: [PATCH 152/170] fix: Update code --- .../dhis/analytics/table/JdbcEventAnalyticsTableManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 577ca3925f8a..458ebe972669 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -243,13 +243,13 @@ private List getLatestAnalyticsTables(AnalyticsTableUpdateParams tables.add(table); log.info( - "Added latest event analytics partition for program: '{}' with start: '{}' and end: '{}'", + "Added latest event analytics partition for program: '{}', start: '{}' and end: '{}'", program.getUid(), toLongDate(startDate), toLongDate(endDate)); } else { log.info( - "No updated latest event data found for program: '{}' with start: '{}' and end: '{}", + "No updated latest event data found for program: '{}', start: '{}' and end: '{}", program.getUid(), toLongDate(lastAnyTableUpdate), toLongDate(endDate)); From 58a8771e417f7947062831d7e7c793ae448576cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 16 Dec 2024 14:17:32 +0100 Subject: [PATCH 153/170] fix: Update code --- .../table/AbstractEventJdbcTableManager.java | 13 ------------- .../analytics/table/AbstractJdbcTableManager.java | 13 +++++++++++++ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 508761d83d12..074febf2d48e 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -45,7 +45,6 @@ import org.hisp.dhis.analytics.table.model.Skip; import org.hisp.dhis.analytics.table.setting.AnalyticsTableSettings; import org.hisp.dhis.category.CategoryService; -import org.hisp.dhis.common.IdentifiableObject; import org.hisp.dhis.common.IdentifiableObjectManager; import org.hisp.dhis.common.ValueType; import org.hisp.dhis.dataapproval.DataApprovalLevelService; @@ -295,16 +294,4 @@ protected String getAttributeValueJoinClause(Program program) { .map(attribute -> replaceQualify(template, toVariableMap(attribute))) .collect(Collectors.joining()); } - - /** - * Returns a map of identifiable properties and values. - * - * @param object the {@link IdentifiableObject}. - * @return a {@link Map}. - */ - protected Map toVariableMap(IdentifiableObject object) { - return Map.of( - "id", String.valueOf(object.getId()), - "uid", quote(object.getUid())); - } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java index d59263be6dad..732c93737af6 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java @@ -62,6 +62,7 @@ import org.hisp.dhis.calendar.Calendar; import org.hisp.dhis.category.CategoryService; import org.hisp.dhis.common.DimensionalObject; +import org.hisp.dhis.common.IdentifiableObject; import org.hisp.dhis.common.IdentifiableObjectManager; import org.hisp.dhis.common.ValueType; import org.hisp.dhis.commons.collection.ListUtils; @@ -452,6 +453,18 @@ protected void invokeTimeAndLog(String sql, String logPattern, Object... args) { log.info("{} in: {}", logMessage, timer.stop().toString()); } + /** + * Returns a map of identifiable properties and values. + * + * @param object the {@link IdentifiableObject}. + * @return a {@link Map}. + */ + protected Map toVariableMap(IdentifiableObject object) { + return Map.of( + "id", String.valueOf(object.getId()), + "uid", quote(object.getUid())); + } + /** * Filters out analytics table columns which were created after the time of the last successful * resource table update. This so that the create table query does not refer to columns not From 79b1892b52e8ce403090262a2a9d0b983e50e06f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 16 Dec 2024 15:43:43 +0100 Subject: [PATCH 154/170] fix: Update code --- .../table/AbstractEventJdbcTableManager.java | 1 - .../dhis/db/sql/ClickHouseSqlBuilder.java | 12 ++++++++++ .../org/hisp/dhis/db/sql/DorisSqlBuilder.java | 16 +++++++++++-- .../hisp/dhis/db/sql/PostgreSqlBuilder.java | 16 +++++++++++-- .../java/org/hisp/dhis/db/sql/SqlBuilder.java | 23 ++++++++++++++++--- .../dhis/db/sql/ClickHouseSqlBuilderTest.java | 16 +++++++++++++ .../hisp/dhis/db/sql/DorisSqlBuilderTest.java | 20 ++++++++++++++++ .../dhis/db/sql/PostgreSqlBuilderTest.java | 20 ++++++++++++++++ 8 files changed, 116 insertions(+), 8 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 074febf2d48e..28d26e34a33c 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -31,7 +31,6 @@ import static org.hisp.dhis.analytics.util.AnalyticsUtils.getColumnType; import static org.hisp.dhis.db.model.DataType.GEOMETRY; import static org.hisp.dhis.db.model.DataType.TEXT; - import java.util.ArrayList; import java.util.List; import java.util.Map; diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java index a5f6976377c5..061cd193a178 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java @@ -259,6 +259,18 @@ public String ifThenElse(String condition, String resultA, String resultB) { return String.format("if(%s, %s, %s)", condition, resultA, resultB); } + @Override + public String ifThenElse( + String conditionA, + String thenResultA, + String conditionB, + String thenResultB, + String elseResult) { + return String.format( + "multiIf(%s, %s, %s, %s, %s)", + conditionA, thenResultA, conditionB, thenResultB, elseResult); + } + // Statements @Override diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java index 741b98099147..0e90cf60ed1e 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java @@ -264,8 +264,20 @@ public String ifThen(String condition, String result) { } @Override - public String ifThenElse(String condition, String resultA, String resultB) { - return String.format("case when %s then %s else %s end", condition, resultA, resultB); + public String ifThenElse(String condition, String thenResult, String elseResult) { + return String.format("case when %s then %s else %s end", condition, thenResult, elseResult); + } + + @Override + public String ifThenElse( + String conditionA, + String thenResultA, + String conditionB, + String thenResultB, + String elseResult) { + return String.format( + "case when %s then %s when %s then %s else %s end", + conditionA, thenResultA, conditionB, thenResultB, elseResult); } // Statements diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgreSqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgreSqlBuilder.java index b19fbfeb1287..21166696f930 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgreSqlBuilder.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgreSqlBuilder.java @@ -291,8 +291,20 @@ public String ifThen(String condition, String result) { } @Override - public String ifThenElse(String condition, String resultA, String resultB) { - return String.format("case when %s then %s else %s end", condition, resultA, resultB); + public String ifThenElse(String condition, String thenResult, String elseResult) { + return String.format("case when %s then %s else %s end", condition, thenResult, elseResult); + } + + @Override + public String ifThenElse( + String conditionA, + String thenResultA, + String conditionB, + String thenResultB, + String elseResult) { + return String.format( + "case when %s then %s when %s then %s else %s end", + conditionA, thenResultA, conditionB, thenResultB, elseResult); } // Statements diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java index 5d2837c57e96..457853313314 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java @@ -342,11 +342,28 @@ public interface SqlBuilder { * Returns a conditional statement. * * @param condition the condition to evaluate. - * @param resultA the result to return if the condition is true. - * @param resultB the result to return if the condition is false. + * @param thenResult the result to return if the condition is true. + * @param elseResult the result to return if the condition is false. * @return a conditional statement. */ - String ifThenElse(String condition, String resultA, String resultB); + String ifThenElse(String condition, String thenResult, String elseResult); + + /** + * Returns a conditional statement. + * + * @param conditionA the first condition to evaluate. + * @param thenResultA the result to return if the first condition is true. + * @param conditionB the second condition to evaluate. + * @param thenResultB the result to return if the second condition is false. + * @param elseResult the result to return if all conditions are false. + * @return a conditional statement. + */ + String ifThenElse( + String conditionA, + String thenResultA, + String conditionB, + String thenResultB, + String elseResult); // Statements diff --git a/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilderTest.java b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilderTest.java index 9e26f64114a3..6d72e9055d70 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilderTest.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilderTest.java @@ -251,6 +251,22 @@ void testIfThenElse() { sqlBuilder.ifThenElse("a.status = 'COMPLETE'", "a.eventdate", "a.scheduleddate")); } + @Test + void testIfThenElseMulti() { + String expected = + """ + multiIf(a.status = 'COMPLETE', a.eventdate, a.status = 'SCHEDULED', a.scheduleddate, a.incidentdate)"""; + + assertEquals( + expected, + sqlBuilder.ifThenElse( + "a.status = 'COMPLETE'", + "a.eventdate", + "a.status = 'SCHEDULED'", + "a.scheduleddate", + "a.incidentdate")); + } + // Statements @Test diff --git a/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java index d914ebdf1346..ba44eef28b6d 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java @@ -235,6 +235,26 @@ void testIfThenElse() { sqlBuilder.ifThenElse("a.status = 'COMPLETE'", "a.eventdate", "a.scheduleddate")); } + @Test + void testIfThenElseMulti() { + String expected = + """ + case \ + when a.status = 'COMPLETE' then a.eventdate \ + when a.status = 'SCHEDULED' then a.scheduleddate \ + else a.incidentdate \ + end"""; + + assertEquals( + expected, + sqlBuilder.ifThenElse( + "a.status = 'COMPLETE'", + "a.eventdate", + "a.status = 'SCHEDULED'", + "a.scheduleddate", + "a.incidentdate")); + } + // Statements @Test diff --git a/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/PostgreSqlBuilderTest.java b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/PostgreSqlBuilderTest.java index 935336050c2b..3a7ff2bd73b5 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/PostgreSqlBuilderTest.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/PostgreSqlBuilderTest.java @@ -261,6 +261,26 @@ void testIfThenElse() { sqlBuilder.ifThenElse("a.status = 'COMPLETE'", "a.eventdate", "a.scheduleddate")); } + @Test + void testIfThenElseMulti() { + String expected = + """ + case \ + when a.status = 'COMPLETE' then a.eventdate \ + when a.status = 'SCHEDULED' then a.scheduleddate \ + else a.incidentdate \ + end"""; + + assertEquals( + expected, + sqlBuilder.ifThenElse( + "a.status = 'COMPLETE'", + "a.eventdate", + "a.status = 'SCHEDULED'", + "a.scheduleddate", + "a.incidentdate")); + } + // Statements @Test From 16b5d816ec3e505902f5d13c57d564c398c0661e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 16 Dec 2024 15:44:11 +0100 Subject: [PATCH 155/170] fix: Update code --- .../hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java | 1 + 1 file changed, 1 insertion(+) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 28d26e34a33c..074febf2d48e 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -31,6 +31,7 @@ import static org.hisp.dhis.analytics.util.AnalyticsUtils.getColumnType; import static org.hisp.dhis.db.model.DataType.GEOMETRY; import static org.hisp.dhis.db.model.DataType.TEXT; + import java.util.ArrayList; import java.util.List; import java.util.Map; From 04bbdf5c5985168a348366168132cdba10e882ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 16 Dec 2024 15:50:16 +0100 Subject: [PATCH 156/170] fix: Update code --- .../analytics/table/AbstractEventJdbcTableManager.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 074febf2d48e..0e98417748f6 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -120,9 +120,8 @@ protected String getColumnExpression(ValueType valueType, String columnExpressio } else if (valueType.isInteger()) { return getCastExpression(columnExpression, NUMERIC_REGEXP, sqlBuilder.dataTypeBigInt()); } else if (valueType.isBoolean()) { - return String.format( - "case when %1$s = 'true' then 1 when %1$s = 'false' then 0 else null end", - columnExpression); + return sqlBuilder.ifThenElse( + columnExpression + " = 'true", "1", columnExpression + " = 'false'", "0", "null"); } else if (valueType.isDate()) { return getCastExpression(columnExpression, DATE_REGEXP, sqlBuilder.dataTypeTimestamp()); } else if (valueType.isGeo() && isSpatialSupport()) { @@ -146,8 +145,9 @@ protected String getColumnExpression(ValueType valueType, String columnExpressio */ protected String getCastExpression(String columnExpression, String filterRegex, String dataType) { String filter = sqlBuilder.regexpMatch(columnExpression, filterRegex); - return String.format( - "case when %s then cast(%s as %s) else null end", filter, columnExpression, dataType); + String result = String.format("cast(%s as %s)", columnExpression, dataType); + + return sqlBuilder.ifThen(filter, result); } @Override From dac941fb06c9c13c919db94b3e614d140e239787 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 16 Dec 2024 16:43:40 +0100 Subject: [PATCH 157/170] fix: Update code --- .../analytics/table/AbstractEventJdbcTableManager.java | 2 +- .../table/AbstractEventJdbcTableManagerTest.java | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 0e98417748f6..f805967cbae5 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -121,7 +121,7 @@ protected String getColumnExpression(ValueType valueType, String columnExpressio return getCastExpression(columnExpression, NUMERIC_REGEXP, sqlBuilder.dataTypeBigInt()); } else if (valueType.isBoolean()) { return sqlBuilder.ifThenElse( - columnExpression + " = 'true", "1", columnExpression + " = 'false'", "0", "null"); + columnExpression + " = 'true'", "1", columnExpression + " = 'false'", "0", "null"); } else if (valueType.isDate()) { return getCastExpression(columnExpression, DATE_REGEXP, sqlBuilder.dataTypeTimestamp()); } else if (valueType.isGeo() && isSpatialSupport()) { diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManagerTest.java index a4ea5e5fa65f..b26200ffb396 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManagerTest.java @@ -74,7 +74,7 @@ void testGetCastExpression() { """ case when eventdatavalues #>> '{GieVkTxp4HH, value}' ~* '^(-?[0-9]+)(\\.[0-9]+)?$' \ then cast(eventdatavalues #>> '{GieVkTxp4HH, value}' as double precision) \ - else null end"""; + end"""; String actual = manager.getCastExpression( @@ -91,7 +91,7 @@ void testGetSelectExpressionNumber() { """ case when eventdatavalues #>> '{GieVkTxp4HH, value}' ~* '^(-?[0-9]+)(\\.[0-9]+)?$' \ then cast(eventdatavalues #>> '{GieVkTxp4HH, value}' as double precision) \ - else null end"""; + end"""; String actual = manager.getColumnExpression(ValueType.NUMBER, "eventdatavalues #>> '{GieVkTxp4HH, value}'"); @@ -103,7 +103,9 @@ then cast(eventdatavalues #>> '{GieVkTxp4HH, value}' as double precision) \ void testGetSelectExpressionBoolean() { String expected = """ - case when eventdatavalues #>> '{Xl3voRRcmpo, value}' = 'true' then 1 when eventdatavalues #>> '{Xl3voRRcmpo, value}' = 'false' then 0 else null end"""; + case when eventdatavalues #>> '{Xl3voRRcmpo, value}' = 'true' then 1 \ + when eventdatavalues #>> '{Xl3voRRcmpo, value}' = 'false' then 0 \ + else null end"""; String actual = manager.getColumnExpression( @@ -118,7 +120,7 @@ void testGetSelectExpressionDate() { """ case when eventdatavalues #>> '{AL04Wbutskk, value}' ~* '^\\d{4}-\\d{2}-\\d{2}(\\s|T)?((\\d{2}:)(\\d{2}:)?(\\d{2}))?(|.(\\d{3})|.(\\d{3})Z)?$' \ then cast(eventdatavalues #>> '{AL04Wbutskk, value}' as timestamp) \ - else null end"""; + end"""; String actual = manager.getColumnExpression(ValueType.DATE, "eventdatavalues #>> '{AL04Wbutskk, value}'"); From 99502fe2a6bc0af03cb2123fa30634d19c1a9d55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 16 Dec 2024 16:45:19 +0100 Subject: [PATCH 158/170] fix: Update code --- .../table/JdbcEventAnalyticsTableManagerDorisTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java index 5b2bca0f3ec1..3e9a5a2a67dd 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java @@ -162,16 +162,16 @@ void verifyGetTableWithDataElements() { String aliasA = "json_unquote(json_extract(eventdatavalues, '$.%s.value')) as `%s`"; String aliasB = - "case when json_unquote(json_extract(eventdatavalues, '$.%s.value')) regexp '^(-?[0-9]+)(\\.[0-9]+)?$' then cast(json_unquote(json_extract(eventdatavalues, '$.%s.value')) as double) else null end as `%s`"; + "case when json_unquote(json_extract(eventdatavalues, '$.%s.value')) regexp '^(-?[0-9]+)(\\.[0-9]+)?$' then cast(json_unquote(json_extract(eventdatavalues, '$.%s.value')) as double) end as `%s`"; String aliasC = "case when json_unquote(json_extract(eventdatavalues, '$.%s.value')) = 'true' then 1 when json_unquote(json_extract(eventdatavalues, '$.%s.value')) = 'false' then 0 else null end as `%s`"; String aliasD = - "case when json_unquote(json_extract(eventdatavalues, '$.%s.value')) regexp '^\\d{4}-\\d{2}-\\d{2}(\\s|T)?((\\d{2}:)(\\d{2}:)?(\\d{2}))?(|.(\\d{3})|.(\\d{3})Z)?$' then cast(json_unquote(json_extract(eventdatavalues, '$.%s.value')) as datetime) else null end as `%s`"; + "case when json_unquote(json_extract(eventdatavalues, '$.%s.value')) regexp '^\\d{4}-\\d{2}-\\d{2}(\\s|T)?((\\d{2}:)(\\d{2}:)?(\\d{2}))?(|.(\\d{3})|.(\\d{3})Z)?$' then cast(json_unquote(json_extract(eventdatavalues, '$.%s.value')) as datetime) end as `%s`"; String aliasE = "json_unquote(json_extract(eventdatavalues, '$.%s.value')) as `%s`"; String aliasF = "(select ou.name from dhis2.public.`organisationunit` ou where ou.uid = json_unquote(json_extract(eventdatavalues, '$.%s.value'))) as `%s`"; String aliasG = - "case when json_unquote(json_extract(eventdatavalues, '$.%s.value')) regexp '^(-?[0-9]+)(\\.[0-9]+)?$' then cast(json_unquote(json_extract(eventdatavalues, '$.%s.value')) as bigint) else null end as `%s`"; + "case when json_unquote(json_extract(eventdatavalues, '$.%s.value')) regexp '^(-?[0-9]+)(\\.[0-9]+)?$' then cast(json_unquote(json_extract(eventdatavalues, '$.%s.value')) as bigint) end as `%s`"; AnalyticsTableUpdateParams params = AnalyticsTableUpdateParams.newBuilder() From 8716f4ba2c4ac79de899f2aea87cef4d88b22f6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 16 Dec 2024 16:47:17 +0100 Subject: [PATCH 159/170] fix: Update code --- .../analytics/table/JdbcEventAnalyticsTableManagerTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java index 15e1eb3b9a6d..7bc0ca4c0208 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java @@ -378,7 +378,7 @@ void verifyGetTableWithDataElements() { String aliasD2 = """ case when eventdatavalues #>> '{deabcdefghP, value}' ~* '^(-?[0-9]+)(\\.[0-9]+)?$' \ - then cast(eventdatavalues #>> '{deabcdefghP, value}' as double precision) else null end as "deabcdefghP\""""; + then cast(eventdatavalues #>> '{deabcdefghP, value}' as double precision) end as "deabcdefghP\""""; String aliasD3 = """ case when eventdatavalues #>> '{deabcdefghY, value}' = 'true' then 1 \ @@ -386,13 +386,13 @@ void verifyGetTableWithDataElements() { String aliasD4 = """ case when eventdatavalues #>> '{deabcdefghW, value}' ~* '^\\d{4}-\\d{2}-\\d{2}(\\s|T)?((\\d{2}:)(\\d{2}:)?(\\d{2}))?(|.(\\d{3})|.(\\d{3})Z)?$' \ - then cast(eventdatavalues #>> '{deabcdefghW, value}' as timestamp) else null end as "deabcdefghW\""""; + then cast(eventdatavalues #>> '{deabcdefghW, value}' as timestamp) end as "deabcdefghW\""""; String aliasD5 = "eventdatavalues #>> '{" + d5.getUid() + ", value}' as \"" + d5.getUid() + "\""; String aliasD6 = """ case when eventdatavalues #>> '{deabcdefghH, value}' ~* '^(-?[0-9]+)(\\.[0-9]+)?$' \ - then cast(eventdatavalues #>> '{deabcdefghH, value}' as bigint) else null end as "deabcdefghH\""""; + then cast(eventdatavalues #>> '{deabcdefghH, value}' as bigint) end as "deabcdefghH\""""; String aliasD7 = """ ST_GeomFromGeoJSON('{\"type\":\"Point\", \"coordinates\":' || (eventdatavalues #>> '{deabcdefghU, value}') || ', "crs":{"type":"name", "properties":{"name":"EPSG:4326"}}}') as "deabcdefghU\""""; From ddf2e5517251c9e9c6b899b32ad2e0a6b4e25909 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 16 Dec 2024 19:28:58 +0100 Subject: [PATCH 160/170] fix: Update code --- .../analytics/table/JdbcEventAnalyticsTableManager.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 458ebe972669..3baf44e2095e 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -41,7 +41,6 @@ import static org.hisp.dhis.system.util.MathUtils.NUMERIC_LENIENT_REGEXP; import static org.hisp.dhis.util.DateUtils.toLongDate; import static org.hisp.dhis.util.DateUtils.toMediumDate; - import java.time.Year; import java.util.ArrayList; import java.util.Collection; @@ -50,7 +49,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import lombok.extern.slf4j.Slf4j; import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.AnalyticsTableType; import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; @@ -88,6 +86,7 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.util.Assert; +import lombok.extern.slf4j.Slf4j; /** * @author Lars Helge Overland @@ -706,11 +705,11 @@ private List getColumnFromDataElementWithLegendSet( * @return a data filter clause. */ private String getDataFilterClause(DataElement dataElement) { - String uid = dataElement.getUid(); ValueType valueType = dataElement.getValueType(); if (valueType.isNumeric() || valueType.isDate()) { - String jsonExpression = sqlBuilder.jsonExtractNested("eventdatavalues", uid, "value"); + String jsonExpression = + sqlBuilder.jsonExtractNested("eventdatavalues", dataElement.getUid(), "value"); String regex = valueType.isNumeric() ? NUMERIC_REGEXP : DATE_REGEXP; return " and " + sqlBuilder.regexpMatch(jsonExpression, regex); From cfc6e938dc0cafb2d9ac1d085a5cbfe8e24225fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 16 Dec 2024 19:29:22 +0100 Subject: [PATCH 161/170] fix: Update code --- .../dhis/analytics/table/JdbcEventAnalyticsTableManager.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 3baf44e2095e..c935ca6c737d 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -41,6 +41,7 @@ import static org.hisp.dhis.system.util.MathUtils.NUMERIC_LENIENT_REGEXP; import static org.hisp.dhis.util.DateUtils.toLongDate; import static org.hisp.dhis.util.DateUtils.toMediumDate; + import java.time.Year; import java.util.ArrayList; import java.util.Collection; @@ -49,6 +50,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import lombok.extern.slf4j.Slf4j; import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.AnalyticsTableType; import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; @@ -86,7 +88,6 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.util.Assert; -import lombok.extern.slf4j.Slf4j; /** * @author Lars Helge Overland @@ -708,7 +709,7 @@ private String getDataFilterClause(DataElement dataElement) { ValueType valueType = dataElement.getValueType(); if (valueType.isNumeric() || valueType.isDate()) { - String jsonExpression = + String jsonExpression = sqlBuilder.jsonExtractNested("eventdatavalues", dataElement.getUid(), "value"); String regex = valueType.isNumeric() ? NUMERIC_REGEXP : DATE_REGEXP; From cf566ec4c0e1a032baf8776107697d81da49c0ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 16 Dec 2024 21:09:34 +0100 Subject: [PATCH 162/170] fix: Update code --- .../table/JdbcEventAnalyticsTableManager.java | 6 +++--- .../org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java | 6 +++--- .../java/org/hisp/dhis/db/sql/DorisSqlBuilder.java | 6 +++--- .../java/org/hisp/dhis/db/sql/PostgreSqlBuilder.java | 5 +++-- .../main/java/org/hisp/dhis/db/sql/SqlBuilder.java | 11 ++++++----- .../hisp/dhis/db/sql/ClickHouseSqlBuilderTest.java | 9 ++++++--- .../org/hisp/dhis/db/sql/DorisSqlBuilderTest.java | 9 ++++++--- .../org/hisp/dhis/db/sql/PostgreSqlBuilderTest.java | 9 ++++++--- 8 files changed, 36 insertions(+), 25 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index c935ca6c737d..2a07e0b69b89 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -489,7 +489,7 @@ private List getColumnForDataElement( DataType dataType = getColumnType(dataElement.getValueType(), isSpatialSupport()); String jsonExpression = - sqlBuilder.jsonExtractNested("eventdatavalues", dataElement.getUid(), "value"); + sqlBuilder.jsonExtract("eventdatavalues", dataElement.getUid(), "value"); String columnExpression = getColumnExpression(dataElement.getValueType(), jsonExpression); String dataFilterClause = getDataFilterClause(dataElement); String selectExpression = getSelectExpression(dataElement, columnExpression); @@ -576,7 +576,7 @@ private String getOrgUnitSelectSubquery(DataElement dataElement, String column) (select ou.${column} from ${organisationunit} ou \ where ou.uid = ${columnExpression}) as ${alias}"""; String columnExpression = - sqlBuilder.jsonExtractNested("eventdatavalues", dataElement.getUid(), "value"); + sqlBuilder.jsonExtract("eventdatavalues", dataElement.getUid(), "value"); String alias = quote(dataElement.getUid()); return replaceQualify( @@ -710,7 +710,7 @@ private String getDataFilterClause(DataElement dataElement) { if (valueType.isNumeric() || valueType.isDate()) { String jsonExpression = - sqlBuilder.jsonExtractNested("eventdatavalues", dataElement.getUid(), "value"); + sqlBuilder.jsonExtract("eventdatavalues", dataElement.getUid(), "value"); String regex = valueType.isNumeric() ? NUMERIC_REGEXP : DATE_REGEXP; return " and " + sqlBuilder.regexpMatch(jsonExpression, regex); diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java index 061cd193a178..be764f22151e 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java @@ -224,9 +224,9 @@ public String jsonExtract(String json, String property) { } @Override - public String jsonExtractNested(String json, String... expression) { - String path = String.join(".", expression); - return String.format("JSONExtractString(%s, '%s')", json, path); + public String jsonExtract(String json, String key, String property) { + String path = String.format("JSONExtractRaw(%s, '%s')", json, key); + return String.format("JSONExtractString(%s, '%s')", path, property); } @Override diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java index 0e90cf60ed1e..5797a81b8502 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java @@ -227,9 +227,9 @@ public String jsonExtract(String json, String property) { } @Override - public String jsonExtractNested(String column, String... expression) { - String path = "$." + String.join(".", expression); - return String.format("json_unquote(json_extract(%s, '%s'))", column, path); + public String jsonExtract(String json, String key, String property) { + String path = "$." + String.join(".", key, property); + return String.format("json_unquote(json_extract(%s, '%s'))", json, path); } @Override diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgreSqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgreSqlBuilder.java index 21166696f930..ade5a4845a38 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgreSqlBuilder.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgreSqlBuilder.java @@ -246,8 +246,9 @@ public String jsonExtract(String column, String property) { } @Override - public String jsonExtractNested(String column, String... expression) { - return String.format("%s #>> '{%s}'", column, String.join(", ", expression)); + public String jsonExtract(String json, String key, String property) { + String path = String.join(", ", key, property); + return String.format("%s #>> '{%s}'", json, path); } @Override diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java index 457853313314..0387241cecfd 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java @@ -284,19 +284,20 @@ public interface SqlBuilder { * Extracts a value from a JSON column using a specified property path. * * @param json the JSON column name or value to extract from. - * @param property the JSON property path to extract. + * @param property the JSON property to extract. * @return the SQL function for JSON value extraction. */ String jsonExtract(String json, String property); /** - * Extracts a nested value from a JSON column. + * Extracts a nested value from a JSON object. * - * @param json the JSON column name or value to extract from. - * @param expression the hierarchical path expression to the nested value. + * @param json the JSON column name or object to extract from. + * @param key the object key. + * @param property the JSON property to extract. * @return a SQL expression to extract the specified nested value from the JSON column. */ - String jsonExtractNested(String json, String... expression); + String jsonExtract(String json, String key, String property); /** * Generates a SQL casting expression for the given column or expression. diff --git a/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilderTest.java b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilderTest.java index 6d72e9055d70..b2b04e284bce 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilderTest.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilderTest.java @@ -231,10 +231,13 @@ void testJsonExtract() { } @Test - void testJsonExtractNested() { + void testJsonExtractObject() { assertEquals( - "JSONExtractString(eventdatavalues, 'D7m8vpzxHDJ.value')", - sqlBuilder.jsonExtractNested("eventdatavalues", "D7m8vpzxHDJ", "value")); + "JSONExtractString(JSONExtractRaw(ev.eventdatavalues, 'D7m8vpzxHDJ'), 'value')", + sqlBuilder.jsonExtract("ev.eventdatavalues", "D7m8vpzxHDJ", "value")); + assertEquals( + "JSONExtractString(JSONExtractRaw(ev.eventdatavalues, 'qrur9Dvnyt5'), 'value')", + sqlBuilder.jsonExtract("ev.eventdatavalues", "qrur9Dvnyt5", "value")); } @Test diff --git a/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java index ba44eef28b6d..2d2e57e98081 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java @@ -215,10 +215,13 @@ void testJsonExtract() { } @Test - void testJsonExtractNested() { + void testJsonExtractObject() { assertEquals( - "json_unquote(json_extract(eventdatavalues, '$.D7m8vpzxHDJ.value'))", - sqlBuilder.jsonExtractNested("eventdatavalues", "D7m8vpzxHDJ", "value")); + "json_unquote(json_extract(ev.eventdatavalues, '$.D7m8vpzxHDJ.value'))", + sqlBuilder.jsonExtract("ev.eventdatavalues", "D7m8vpzxHDJ", "value")); + assertEquals( + "json_unquote(json_extract(ev.eventdatavalues, '$.qrur9Dvnyt5.value'))", + sqlBuilder.jsonExtract("ev.eventdatavalues", "qrur9Dvnyt5", "value")); } @Test diff --git a/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/PostgreSqlBuilderTest.java b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/PostgreSqlBuilderTest.java index 3a7ff2bd73b5..8c4516f901cf 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/PostgreSqlBuilderTest.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/PostgreSqlBuilderTest.java @@ -241,10 +241,13 @@ void testJsonExtract() { } @Test - void testJsonExtractNested() { + void testJsonExtractObject() { assertEquals( - "eventdatavalues #>> '{D7m8vpzxHDJ, value}'", - sqlBuilder.jsonExtractNested("eventdatavalues", "D7m8vpzxHDJ", "value")); + "ev.eventdatavalues #>> '{D7m8vpzxHDJ, value}'", + sqlBuilder.jsonExtract("ev.eventdatavalues", "D7m8vpzxHDJ", "value")); + assertEquals( + "ev.eventdatavalues #>> '{qrur9Dvnyt5, value}'", + sqlBuilder.jsonExtract("ev.eventdatavalues", "qrur9Dvnyt5", "value")); } @Test From d94712ebf6142100fc4ac845e271c15393accf91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 16 Dec 2024 22:16:37 +0100 Subject: [PATCH 163/170] fix: Update code --- .../java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java | 6 +++++- .../java/org/hisp/dhis/db/sql/ClickHouseSqlBuilderTest.java | 3 ++- .../test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java | 1 + .../java/org/hisp/dhis/db/sql/PostgreSqlBuilderTest.java | 1 + 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java index be764f22151e..a94a0b299df8 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java @@ -125,9 +125,13 @@ public String dataTypeGeometryPoint() { return "String"; } + /** + * ClickHouse JSON data type is in beta and is not yet production-ready. Check back and enable + * JSON when available. + */ @Override public String dataTypeJson() { - return "JSON"; + return "String"; } // Index functions diff --git a/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilderTest.java b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilderTest.java index b2b04e284bce..96bd70b71a7b 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilderTest.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilderTest.java @@ -101,6 +101,7 @@ private Table getTableD() { void testDataTypes() { assertEquals("Float64", sqlBuilder.dataTypeDouble()); assertEquals("DateTime64(3)", sqlBuilder.dataTypeTimestamp()); + assertEquals("String", sqlBuilder.dataTypeJson()); } // Index types @@ -279,7 +280,7 @@ void testCreateTableA() { String expected = """ create table "immunization" ("id" Int64 not null,"data" String not null,\ - "period" String not null,"created" DateTime64(3) null,"user" JSON null,\ + "period" String not null,"created" DateTime64(3) null,"user" String null,\ "value" Float64 null) \ engine = MergeTree() \ order by ("id");"""; diff --git a/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java index 2d2e57e98081..141fdcc90c1a 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java @@ -86,6 +86,7 @@ private Table getTableC() { void testDataTypes() { assertEquals("double", sqlBuilder.dataTypeDouble()); assertEquals("datetime", sqlBuilder.dataTypeTimestamp()); + assertEquals("json", sqlBuilder.dataTypeJson()); } // Index types diff --git a/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/PostgreSqlBuilderTest.java b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/PostgreSqlBuilderTest.java index 8c4516f901cf..a0cd4e5d3953 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/PostgreSqlBuilderTest.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/PostgreSqlBuilderTest.java @@ -117,6 +117,7 @@ private Table getTableC() { void testDataTypes() { assertEquals("double precision", sqlBuilder.dataTypeDouble()); assertEquals("geometry", sqlBuilder.dataTypeGeometry()); + assertEquals("jsonb", sqlBuilder.dataTypeJson()); } // Index types From 1867eb5bba5c46ef3201239be08dc780101fd4f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 16 Dec 2024 22:33:44 +0100 Subject: [PATCH 164/170] fix: Update code --- .../JdbcOwnershipAnalyticsTableManager.java | 4 +++- .../hibernate/ProgramOwnershipHistory.hbm.xml | 10 ++++------ .../TrackedEntityProgramOwner.hbm.xml | 18 ++++++++---------- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java index 50bf484bddb9..00e13a1c4e47 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java @@ -91,6 +91,7 @@ public class JdbcOwnershipAnalyticsTableManager extends AbstractEventJdbcTableMa AnalyticsTableColumn.builder() .name("teuid") .dataType(CHARACTER_11) + .nullable(NOT_NULL) .selectExpression("te.uid") .build(), AnalyticsTableColumn.builder() @@ -259,7 +260,8 @@ private String getInputSql(Program program) { and exists (\ select 1 from ${programownershiphistory} p \ where o.trackedentityid = p.trackedentityid \ - and p.programid=${programId} and p.organisationunitid is not null)) a \ + and p.programid = ${programId} \ + and p.organisationunitid is not null)) a \ inner join ${trackedentity} te on a.trackedentityid = te.trackedentityid \ inner join ${organisationunit} ou on a.organisationunitid = ou.organisationunitid \ left join analytics_rs_orgunitstructure ous on a.organisationunitid = ous.organisationunitid \ diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/resources/org/hisp/dhis/program/hibernate/ProgramOwnershipHistory.hbm.xml b/dhis-2/dhis-services/dhis-service-core/src/main/resources/org/hisp/dhis/program/hibernate/ProgramOwnershipHistory.hbm.xml index e7e469656c0c..6365e268e39e 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/resources/org/hisp/dhis/program/hibernate/ProgramOwnershipHistory.hbm.xml +++ b/dhis-2/dhis-services/dhis-service-core/src/main/resources/org/hisp/dhis/program/hibernate/ProgramOwnershipHistory.hbm.xml @@ -14,13 +14,11 @@ - + - + diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/resources/org/hisp/dhis/trackedentity/hibernate/TrackedEntityProgramOwner.hbm.xml b/dhis-2/dhis-services/dhis-service-core/src/main/resources/org/hisp/dhis/trackedentity/hibernate/TrackedEntityProgramOwner.hbm.xml index bcb0104a0b40..9f31483898a3 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/resources/org/hisp/dhis/trackedentity/hibernate/TrackedEntityProgramOwner.hbm.xml +++ b/dhis-2/dhis-services/dhis-service-core/src/main/resources/org/hisp/dhis/trackedentity/hibernate/TrackedEntityProgramOwner.hbm.xml @@ -11,22 +11,20 @@ - + + + column="programid" not-null="true" foreign-key="fk_trackedentityprogramowner_programid" + lazy="false" unique-key="uk_tei_program" /> - + From dc2f539ae075b13506032e5c88de74236cbc74fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 16 Dec 2024 22:47:16 +0100 Subject: [PATCH 165/170] fix: Update code --- .../table/JdbcOwnershipAnalyticsTableManager.java | 4 ++-- .../table/JdbcOwnershipAnalyticsTableManagerTest.java | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java index 00e13a1c4e47..e33ad0454aac 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java @@ -251,12 +251,12 @@ private String getInputSql(Program program) { \sfrom (\ select h.trackedentityid, '${historyTableId}' as startdate, h.enddate as enddate, h.organisationunitid \ from ${programownershiphistory} h \ - where h.programid=${programId} \ + where h.programid = ${programId} \ and h.organisationunitid is not null \ union \ select o.trackedentityid, '${trackedEntityOwnTableId}' as startdate, null as enddate, o.organisationunitid \ from ${trackedentityprogramowner} o \ - where o.programid=${programId} \ + where o.programid = ${programId} \ and exists (\ select 1 from ${programownershiphistory} p \ where o.trackedentityid = p.trackedentityid \ diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManagerTest.java index b399999ff371..62de96dba85d 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManagerTest.java @@ -278,14 +278,16 @@ void testPopulateTable() throws SQLException { select te.uid,a.startdate,a.enddate,ou.uid from (\ select h.trackedentityid, '1001-01-01' as startdate, h.enddate as enddate, h.organisationunitid \ from "programownershiphistory" h \ - where h.programid=0 and h.organisationunitid is not null \ + where h.programid = 0 \ + and h.organisationunitid is not null \ union \ select o.trackedentityid, '2002-02-02' as startdate, null as enddate, o.organisationunitid \ from "trackedentityprogramowner" o \ - where o.programid=0 \ + where o.programid = 0 \ and exists (select 1 from "programownershiphistory" p \ where o.trackedentityid = p.trackedentityid \ - and p.programid=0 and p.organisationunitid is not null)\ + and p.programid = 0 \ + and p.organisationunitid is not null)\ ) a \ inner join "trackedentity" te on a.trackedentityid = te.trackedentityid \ inner join "organisationunit" ou on a.organisationunitid = ou.organisationunitid \ From a84bac6734a5fe96391c1c497b60edcb1bddd5ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 17 Dec 2024 11:32:58 +0100 Subject: [PATCH 166/170] fix: Update code --- .../analytics/table/JdbcOwnershipAnalyticsTableManager.java | 5 ++--- .../table/JdbcOwnershipAnalyticsTableManagerTest.java | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java index e33ad0454aac..00b6f8635f50 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java @@ -32,7 +32,6 @@ import static org.hisp.dhis.db.model.DataType.DATE; import static org.hisp.dhis.db.model.constraint.Nullable.NOT_NULL; import static org.hisp.dhis.program.ProgramType.WITHOUT_REGISTRATION; - import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; @@ -41,7 +40,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; -import lombok.extern.slf4j.Slf4j; import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.AnalyticsTableType; import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; @@ -69,6 +67,7 @@ import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import lombok.extern.slf4j.Slf4j; /** * Populates the analytics_ownership_[programuid] table which is joined for tracker analytics @@ -253,7 +252,7 @@ private String getInputSql(Program program) { from ${programownershiphistory} h \ where h.programid = ${programId} \ and h.organisationunitid is not null \ - union \ + union distinct \ select o.trackedentityid, '${trackedEntityOwnTableId}' as startdate, null as enddate, o.organisationunitid \ from ${trackedentityprogramowner} o \ where o.programid = ${programId} \ diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManagerTest.java index 62de96dba85d..20f3f3937096 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManagerTest.java @@ -48,7 +48,6 @@ import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.mockingDetails; import static org.mockito.Mockito.when; - import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; @@ -280,7 +279,7 @@ select te.uid,a.startdate,a.enddate,ou.uid from (\ from "programownershiphistory" h \ where h.programid = 0 \ and h.organisationunitid is not null \ - union \ + union distinct \ select o.trackedentityid, '2002-02-02' as startdate, null as enddate, o.organisationunitid \ from "trackedentityprogramowner" o \ where o.programid = 0 \ From 10b44b736767306d167278121c55184fc6275a77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 17 Dec 2024 11:48:31 +0100 Subject: [PATCH 167/170] fix: Update code --- .../analytics/table/JdbcOwnershipAnalyticsTableManager.java | 3 ++- .../table/JdbcOwnershipAnalyticsTableManagerTest.java | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java index 00b6f8635f50..a6155311f886 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java @@ -32,6 +32,7 @@ import static org.hisp.dhis.db.model.DataType.DATE; import static org.hisp.dhis.db.model.constraint.Nullable.NOT_NULL; import static org.hisp.dhis.program.ProgramType.WITHOUT_REGISTRATION; + import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; @@ -40,6 +41,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; +import lombok.extern.slf4j.Slf4j; import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.AnalyticsTableType; import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; @@ -67,7 +69,6 @@ import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; -import lombok.extern.slf4j.Slf4j; /** * Populates the analytics_ownership_[programuid] table which is joined for tracker analytics diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManagerTest.java index 20f3f3937096..c71192297e17 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManagerTest.java @@ -48,6 +48,7 @@ import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.mockingDetails; import static org.mockito.Mockito.when; + import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; From cf97f37ee15b900803955db8b695b6c4477f629c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 17 Dec 2024 11:58:18 +0100 Subject: [PATCH 168/170] fix: Update code --- .../analytics/table/JdbcOwnershipAnalyticsTableManager.java | 2 +- .../analytics/table/JdbcOwnershipAnalyticsTableManagerTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java index a6155311f886..1c4fb2122024 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java @@ -253,7 +253,7 @@ private String getInputSql(Program program) { from ${programownershiphistory} h \ where h.programid = ${programId} \ and h.organisationunitid is not null \ - union distinct \ + union all \ select o.trackedentityid, '${trackedEntityOwnTableId}' as startdate, null as enddate, o.organisationunitid \ from ${trackedentityprogramowner} o \ where o.programid = ${programId} \ diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManagerTest.java index c71192297e17..afa72fb68c6d 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManagerTest.java @@ -280,7 +280,7 @@ select te.uid,a.startdate,a.enddate,ou.uid from (\ from "programownershiphistory" h \ where h.programid = 0 \ and h.organisationunitid is not null \ - union distinct \ + union all \ select o.trackedentityid, '2002-02-02' as startdate, null as enddate, o.organisationunitid \ from "trackedentityprogramowner" o \ where o.programid = 0 \ From 75644d015c78f5edd797c653055b69551730a032 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 17 Dec 2024 12:13:52 +0100 Subject: [PATCH 169/170] fix: Update code --- .../analytics/table/JdbcOwnershipAnalyticsTableManager.java | 2 +- .../analytics/table/JdbcOwnershipAnalyticsTableManagerTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java index 1c4fb2122024..a6155311f886 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java @@ -253,7 +253,7 @@ private String getInputSql(Program program) { from ${programownershiphistory} h \ where h.programid = ${programId} \ and h.organisationunitid is not null \ - union all \ + union distinct \ select o.trackedentityid, '${trackedEntityOwnTableId}' as startdate, null as enddate, o.organisationunitid \ from ${trackedentityprogramowner} o \ where o.programid = ${programId} \ diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManagerTest.java index afa72fb68c6d..c71192297e17 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManagerTest.java @@ -280,7 +280,7 @@ select te.uid,a.startdate,a.enddate,ou.uid from (\ from "programownershiphistory" h \ where h.programid = 0 \ and h.organisationunitid is not null \ - union all \ + union distinct \ select o.trackedentityid, '2002-02-02' as startdate, null as enddate, o.organisationunitid \ from "trackedentityprogramowner" o \ where o.programid = 0 \ From 5e2ce559386a1e599639795be3ebefc739f5963c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 17 Dec 2024 12:33:07 +0100 Subject: [PATCH 170/170] fix: Update code --- .../table/JdbcOwnershipAnalyticsTableManager.java | 8 ++++---- .../table/JdbcOwnershipAnalyticsTableManagerTest.java | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java index a6155311f886..7054896f8821 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java @@ -257,10 +257,10 @@ private String getInputSql(Program program) { select o.trackedentityid, '${trackedEntityOwnTableId}' as startdate, null as enddate, o.organisationunitid \ from ${trackedentityprogramowner} o \ where o.programid = ${programId} \ - and exists (\ - select 1 from ${programownershiphistory} p \ - where o.trackedentityid = p.trackedentityid \ - and p.programid = ${programId} \ + and o.trackedentityid in (\ + select distinct p.trackedentityid \ + from ${programownershiphistory} p \ + where p.programid = ${programId} \ and p.organisationunitid is not null)) a \ inner join ${trackedentity} te on a.trackedentityid = te.trackedentityid \ inner join ${organisationunit} ou on a.organisationunitid = ou.organisationunitid \ diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManagerTest.java index c71192297e17..d18d0fb6353e 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManagerTest.java @@ -284,11 +284,11 @@ select te.uid,a.startdate,a.enddate,ou.uid from (\ select o.trackedentityid, '2002-02-02' as startdate, null as enddate, o.organisationunitid \ from "trackedentityprogramowner" o \ where o.programid = 0 \ - and exists (select 1 from "programownershiphistory" p \ - where o.trackedentityid = p.trackedentityid \ - and p.programid = 0 \ - and p.organisationunitid is not null)\ - ) a \ + and o.trackedentityid in (\ + select distinct p.trackedentityid \ + from "programownershiphistory" p \ + where p.programid = 0 \ + and p.organisationunitid is not null)) a \ inner join "trackedentity" te on a.trackedentityid = te.trackedentityid \ inner join "organisationunit" ou on a.organisationunitid = ou.organisationunitid \ left join analytics_rs_orgunitstructure ous on a.organisationunitid = ous.organisationunitid \