fix: Remove unnecessary properties from datasetMappings and add new m…

…apping for scientificMetadata query

src/common/utils.ts

@@ -108,7 +108,6 @@ export const mapScientificQuery = (
   const scientificFilterQueryOr: Record<string, unknown>[] = [];
 
   const keyToFieldMapping: Record<string, string> = {
-    scientificMetadata: "scientificMetadata",
     scientific: "scientificMetadata",
     characteristics: "sampleCharacteristics",
   };
@@ -180,6 +179,7 @@ export const mapScientificQuery = (
   } else if (scientificFilterQueryOr.length > 1) {
     scientificFilterQuery["$and"] = scientificFilterQueryOr;
   }
+
   return scientificFilterQuery;
 };
 

src/elastic-search/configuration/datasetFieldMapping.ts

@@ -33,15 +33,7 @@ export const datasetMappings: MappingObject = {
   scientificMetadata: {
     type: "nested",
     dynamic: true,
-    properties: {
-      runNumber: {
-        properties: {
-          value: {
-            type: "long",
-          },
-        },
-      },
-    },
+    properties: {},
   },
   history: {
     type: "nested",

src/elastic-search/configuration/indexSetting.ts

@@ -43,13 +43,25 @@ export const dynamic_template: Record<string, MappingDynamicTemplate>[] = [
       },
     },
   },
+  {
+    double_as_double: {
+      path_match: "scientificMetadata.*.*",
+      match_mapping_type: "double",
+      mapping: {
+        type: "double",
+        coerce: true,
+        ignore_malformed: true,
+      },
+    },
+  },
   {
     date_as_keyword: {
       path_match: "scientificMetadata.*.*",
       match_mapping_type: "date",
       mapping: {
-        type: "keyword",
-        ignore_above: 256,
+        type: "date",
+        format: "strict_date_optional_time||epoch_millis",
+        ignore_malformed: true,
       },
     },
   },

src/elastic-search/elastic-search.service.ts

@@ -26,9 +26,9 @@ import {
 import { ConfigService } from "@nestjs/config";
 import { sleep } from "src/common/utils";
 import {
-  transformKeysInObject,
   initialSyncTransform,
   transformFacets,
+  addValueType,
 } from "./helpers/utils";
 
 import { SortFields } from "./providers/fields.enum";
@@ -362,7 +362,7 @@ export class ElasticSearchService implements OnModuleInit {
   async updateInsertDocument(data: Partial<DatasetDocument>) {
     //NOTE: Replace all keys with lower case, also replace spaces and dot with underscore
     delete data._id;
-    const transformedScientificMetadata = transformKeysInObject(
+    const transformedScientificMetadata = addValueType(
       data.scientificMetadata as Record<string, unknown>,
     );
 

src/elastic-search/helpers/utils.ts

@@ -3,19 +3,24 @@ import {
   AggregationsFrequentItemSetsBucketKeys,
 } from "@elastic/elasticsearch/lib/api/types";
 import { DatasetClass } from "src/datasets/schemas/dataset.schema";
-import { IFilter, ITransformedFullFacets } from "../interfaces/es-common.type";
+import {
+  IFilter,
+  IQuery,
+  ITransformedFullFacets,
+} from "../interfaces/es-common.type";
 
 export const transformKey = (key: string): string => {
   return key.trim().replace(/[.]/g, "\\.").replace(/ /g, "_").toLowerCase();
 };
-export const transformKeysInObject = (obj: Record<string, unknown>) => {
+export const addValueType = (obj: Record<string, unknown>) => {
   const newObj: Record<string, unknown> = {};
 
   for (const [key, value] of Object.entries(obj)) {
     const newKey = transformKey(key);
 
     const isNumberValueType =
       typeof (value as Record<string, unknown>)?.value === "number";
+
     if (isNumberValueType) {
       (value as Record<string, unknown>)["value_type"] = "number";
     } else {
@@ -66,51 +71,100 @@ export const initialSyncTransform = (obj: DatasetClass) => {
   return modifiedDocInObject;
 };
 
-export const convertToElasticSearchQuery = (
-  scientificQuery: Record<string, unknown>,
-) => {
+const extractNestedQueryOperationValue = (query: IQuery) => {
+  const field = Object.keys(query)[0];
+  const operationWithPrefix = Object.keys(query[field])[0];
+
+  const value =
+    typeof query[field][operationWithPrefix] === "string"
+      ? (query[field][operationWithPrefix] as string).trim()
+      : query[field][operationWithPrefix];
+
+  const operation = operationWithPrefix.replace("$", "");
+
+  return { operation, value, field };
+};
+
+export const convertToElasticSearchQuery = (scientificQuery: any) => {
   const filters: IFilter[] = [];
 
   for (const field in scientificQuery) {
-    const query = scientificQuery[field] as Record<string, unknown>;
-    const operation = Object.keys(query)[0];
-    const value =
-      typeof query[operation] === "string"
-        ? (query[operation] as string).trim()
-        : query[operation];
-
-    const esOperation = operation.replace("$", "");
-
-    // NOTE-EXAMPLE:
-    // trasnformedKey = "scientificMetadata.someKey.value"
-    // firstPart = "scientificMetadata",
-    // middlePart = "someKey"
-    const { transformedKey, firstPart, middlePart } = transformMiddleKey(field);
-
-    let filter = {};
-
-    const fieldType = field.split(".").pop();
-
-    if (fieldType === "valueSI" || fieldType === "value") {
-      const numberFilter = {
-        term: {
-          [`${firstPart}.${middlePart}.value_type`]:
-            typeof value === "number" ? "number" : "string",
+    const query = scientificQuery[field] as any;
+
+    if (field === "$and") {
+      query.forEach((query: any) => {
+        const shouldQueries = query.$or.map((orQuery: any) => {
+          const { operation, value, field } =
+            extractNestedQueryOperationValue(orQuery);
+          const filterType = operation === "eq" ? "term" : "range";
+          return {
+            [filterType]: {
+              [field]: operation === "eq" ? value : { [operation]: value },
+            },
+          };
+        });
+        filters.push({
+          bool: {
+            should: shouldQueries,
+            minimum_should_match: 1,
+          },
+        });
+      });
+    } else if (field === "$or") {
+      const shouldQueries = query.map((query: any) => {
+        const { operation, value, field } =
+          extractNestedQueryOperationValue(query);
+        const filterType = operation === "eq" ? "term" : "range";
+        return {
+          [filterType]: {
+            [field]: operation === "eq" ? value : { [operation]: value },
+          },
+        };
+      });
+      filters.push({
+        bool: {
+          should: shouldQueries,
+          minimum_should_match: 1,
         },
-      };
-      filters.push(numberFilter);
+      });
+    } else {
+      const operation = Object.keys(query)[0];
+      const value =
+        typeof query[operation] === "string"
+          ? (query[operation] as string).trim()
+          : query[operation];
+      const esOperation = operation.replace("$", "");
+
+      // NOTE:
+      // trasnformedKey = "scientificMetadata.someKey.value"
+      // firstPart = "scientificMetadata",
+      // middlePart = "someKey"
+      // lastPart = "value"
+      const { transformedKey, firstPart, middlePart, lastPart } =
+        transformMiddleKey(field);
+
+      if (lastPart === "valueSI" || lastPart === "value") {
+        const numberFilter = {
+          term: {
+            [`${firstPart}.${middlePart}.value_type`]:
+              typeof value === "number" ? "number" : "string",
+          },
+        };
+        filters.push(numberFilter);
+      }
+
+      const filter =
+        esOperation === "eq"
+          ? {
+              term: { [`${transformedKey}`]: value },
+            }
+          : {
+              range: { [`${transformedKey}`]: { [esOperation]: value } },
+            };
+
+      filters.push(filter);
     }
-
-    filter =
-      esOperation === "eq"
-        ? {
-            term: { [`${transformedKey}`]: value },
-          }
-        : { range: { [`${transformedKey}`]: { [esOperation]: value } } };
-
-    filters.push(filter);
   }
-
   return filters;
 };
 

src/elastic-search/interfaces/es-common.type.ts

@@ -20,6 +20,12 @@ export interface IShould {
   term?: {
     [key: string]: string | undefined;
   };
+  range?: {
+    [key: string]: {
+      gte?: string | number;
+      lte?: string | number;
+    };
+  };
 }
 
 export interface IBoolShould {
@@ -56,6 +62,10 @@ export interface IFilter {
       };
     };
   };
+  bool?: {
+    should: IShould[];
+    minimum_should_match?: number;
+  };
 }
 
 export interface IFullFacets {
@@ -81,3 +91,8 @@ export interface ITransformedFullFacets {
       }
     | { totalSets: number };
 }
+
+type Operation = "$eq" | "$neq" | "$gte" | "$gt" | "$lte" | "$lt";
+export interface IQuery {
+  [key: string]: any;
+}