Merge pull request #27 from SwissOpenEM/23-metadata-extractor-integra…

…tion

23 metadata extractor integration

.gitignore

@@ -25,6 +25,7 @@ go.work.sum
 .env
 
 **/build/**
+!cmd/openem-ingestor-app/build/appicon.png
 
 dist/**
 

.goreleaser.yaml

@@ -22,7 +22,7 @@ before:
 
 builds:
   # The app build is invoked via wails and binary copied manually to the expected output folder
-  - id: "openem-ingestor-app"
+  - id: openem-ingestor-app-linux
     hooks:
         pre:
         - cmd: mkdir {{dir .Path}}
@@ -38,14 +38,34 @@ builds:
     dir: ./cmd/openem-ingestor-app
     goos:
       - linux
+    goarch:
+      - amd64
+    main: .
+    binary: openem-ingestor-app
+    gobinary: echo 
+
+  - id: openem-ingestor-app-win
+    hooks:
+        pre:
+        - cmd: mkdir {{dir .Path}}
+        post:
+        - dir: cmd/openem-ingestor-app
+          cmd: wails build -platform {{ .Os }} -clean -windowsconsole -ldflags "-s -w  -X 'main.version={{.Version}}'" -trimpath -o {{base .Path}}
+          output: true
+        - dir: cmd/openem-ingestor-app
+          cmd: cp build/bin/{{base .Path}} {{dir .Path}}/
+          output: true
+    env:
+      - CGO_ENABLED=0
+    dir: ./cmd/openem-ingestor-app
+    goos:
       - windows
     goarch:
       - amd64
     main: .
     binary: openem-ingestor-app
     gobinary: echo 
 
-
   - id: "openem-ingestor-service"
     flags:
       - -trimpath
@@ -63,7 +83,8 @@ builds:
 archives:
   - id: "openem-ingestor-app"
     builds: 
-      - openem-ingestor-app
+      - openem-ingestor-app-linux
+      - openem-ingestor-app-win
     format: tar.gz
 
     wrap_in_directory: true
@@ -104,6 +125,7 @@ archives:
       dst: ./
     - src: configs/schemas/*.json
       dst: schemas
+
 changelog:
   sort: asc
   filters:

cmd/openem-ingestor-app/app.go

@@ -2,10 +2,13 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"log"
+	"log/slog"
 
 	core "github.com/SwissOpenEM/Ingestor/internal/core"
-	"github.com/SwissOpenEM/Ingestor/internal/task"
+	metadataextractor "github.com/SwissOpenEM/Ingestor/internal/metadataextractor"
+	task "github.com/SwissOpenEM/Ingestor/internal/task"
 	webserver "github.com/SwissOpenEM/Ingestor/internal/webserver"
 
 	"github.com/google/uuid"
@@ -48,10 +51,11 @@ func (w *WailsNotifier) OnTaskProgress(id uuid.UUID, current_file int, total_fil
 
 // App struct
 type App struct {
-	ctx       context.Context
-	taskqueue core.TaskQueue
-	config    core.Config
-	version   string
+	ctx              context.Context
+	taskqueue        core.TaskQueue
+	config           core.Config
+	extractorHandler *metadataextractor.ExtractorHandler
+	version          string
 }
 
 // NewApp creates a new App application struct
@@ -77,6 +81,9 @@ func (b *App) beforeClose(ctx context.Context) (prevent bool) {
 // so we can call the runtime methods
 func (a *App) startup(ctx context.Context) {
 	a.ctx = ctx
+
+	a.extractorHandler = metadataextractor.NewExtractorHandler(a.config.MetadataExtractors)
+
 	a.taskqueue = core.TaskQueue{Config: a.config,
 		AppContext: a.ctx,
 		Notifier:   &WailsNotifier{AppContext: a.ctx},
@@ -88,11 +95,12 @@ func (a *App) startup(ctx context.Context) {
 		s := webserver.NewIngesterServer(ingestor, port)
 		log.Fatal(s.ListenAndServe())
 	}(a.config.Misc.Port)
+
 }
 
 func (a *App) SelectFolder() {
 	folder, err := task.SelectFolder(a.ctx)
-	if err != nil {
+	if err != nil || folder.FolderPath == "" {
 		return
 	}
 
@@ -102,6 +110,58 @@ func (a *App) SelectFolder() {
 	}
 }
 
+func (a *App) ExtractMetadata(extractor_name string, id uuid.UUID) string {
+
+	folder := a.taskqueue.GetTaskFolder(id)
+	if folder == "" {
+		return ""
+	}
+
+	log_message := func(id uuid.UUID, msg string) {
+		slog.Info("Extractor output: ", "message", msg)
+		runtime.EventsEmit(a.ctx, "log-update", id, msg)
+	}
+
+	log_error := func(id uuid.UUID, msg string) {
+		slog.Info("Extractor error: ", "message", msg)
+		runtime.EventsEmit(a.ctx, "log-update", id, msg)
+	}
+
+	outputfile := metadataextractor.MetadataFilePath(folder)
+
+	metadata, err := a.extractorHandler.ExtractMetadata(a.ctx, extractor_name, folder, outputfile, func(message string) { log_message(id, message) }, func(message string) { log_error(id, message) })
+
+	if err != nil {
+		slog.Error("Metadata extraction failed", "error", err.Error())
+		return fmt.Sprintf("{\"status\":\"%s\"}", err.Error())
+	}
+	return metadata
+}
+
+type ExtractionMethod struct {
+	Name   string
+	Schema string
+}
+
+func (e *ExtractionMethod) GetName() string {
+	return e.Name
+}
+
+func (e *ExtractionMethod) GetSchema() string {
+	return e.Schema
+}
+
+func (a *App) AvailableMethods() []ExtractionMethod {
+	e := []ExtractionMethod{}
+	for _, ex := range a.extractorHandler.AvailableMethods() {
+		e = append(e, ExtractionMethod{
+			Name:   ex.Name,
+			Schema: ex.Schema,
+		})
+	}
+	return e
+}
+
 func (a *App) CancelTask(id uuid.UUID) {
 	a.taskqueue.CancelTask(id)
 }

cmd/openem-ingestor-app/frontend/src/App.svelte

@@ -1,15 +1,22 @@
 <script lang="ts">
   import logo from "./assets/images/logo-wide-1024x317.png";
   import {
+    ExtractMetadata,
     SelectFolder,
     CancelTask,
     RemoveTask,
     ScheduleTask,
+    AvailableMethods,
   } from "../wailsjs/go/main/App.js";
   import { EventsOn } from "../wailsjs/runtime/runtime";
   import List from "./List.svelte";
   import ListElement from "./ListElement.svelte";
 
+  let selected_extractor;
+
+  async function extractMetadata(id: string): Promise<string> {
+    return await ExtractMetadata(selected_extractor, id);
+  }
   function selectFolder(): void {
     SelectFolder();
   }
@@ -38,13 +45,31 @@
       status: "Selected",
       progress: 0,
       component: ListElement,
+      extractMetadata: extractMetadata,
       cancelTask: cancelTask,
       scheduleTask: scheduleTask,
       removeTask: removeTask,
     };
     return id;
   }
 
+  let extractors = ["No extractors found"];
+  let schemas = {};
+
+  async function refreshExtractors() {
+    AvailableMethods().then((a) => {
+      extractors = [];
+      a.forEach((element) => {
+        extractors.push(element.Name);
+        schemas[element.Name] = atob(element.Schema);
+      });
+      if (extractors.length > 0) selected_extractor = extractors[0];
+      else selected_extractor = ["No extractors found"];
+    });
+  }
+
+  window.onload = refreshExtractors;
+
   EventsOn("folder-added", (id, folder) => {
     newItem(id, folder);
   });
@@ -74,18 +99,46 @@
     items[id].status = "Canceled";
     items = items;
   });
+
+  EventsOn("log-update", (id, message) => {
+    console.log(id);
+    items[id].status += "\n" + message;
+    items = items;
+  });
+
   EventsOn(
     "progress-update",
     (id, current_file, total_files, elapsed_seconds) => {
       const perc = (parseFloat(current_file) / parseFloat(total_files)) * 100;
       items[id].progress = perc.toFixed(0);
-      items[id].status = "Uploading... " + secondsToStr(elapsed_seconds);
+      items[id].status +=
+        "\n" + "Uploading... " + secondsToStr(elapsed_seconds);
     },
   );
 </script>
 
 <main>
-  <img alt="OpenEM logo" id="logo" src={logo} />
+  <img alt="OpenEM logo" id="logo" src={logo} height="200px" />
+
+  <div>
+    <h3>Metadata Extractors</h3>
+    <select bind:value={selected_extractor}>
+      {#each extractors as extractor}
+        <option value={extractor}>
+          {extractor}
+        </option>
+      {/each}
+    </select>
+    <button class="btn" on:click={refreshExtractors}> Refresh </button>
+  </div>
+  <div>
+    <textarea
+      style="height:200px; width:400px"
+      bind:value={schemas[selected_extractor]}
+    />
+  </div>
+
+  <h3>Datasets</h3>
   <button class="btn" on:click={selectFolder}>Select Folder</button>
   <div>
     <div id="upload-list">
@@ -97,56 +150,13 @@
 <style>
   #logo {
     display: block;
-    width: 50%;
-    height: 50%;
+    width: 20%;
+    height: 20%;
     margin: auto;
     padding: 10% 0 0;
     background-position: center;
     background-repeat: no-repeat;
     background-size: 100% 100%;
     background-origin: content-box;
   }
-
-  .result {
-    height: 20px;
-    line-height: 20px;
-    margin: 1.5rem auto;
-  }
-
-  .input-box .btn {
-    width: 60px;
-    height: 30px;
-    line-height: 30px;
-    border-radius: 3px;
-    border: none;
-    margin: 0 0 0 20px;
-    padding: 0 8px;
-    cursor: pointer;
-  }
-
-  .input-box .btn:hover {
-    background-image: linear-gradient(to top, #cfd9df 0%, #e2ebf0 100%);
-    color: #333333;
-  }
-
-  .input-box .input {
-    border: none;
-    border-radius: 3px;
-    outline: none;
-    height: 30px;
-    line-height: 30px;
-    padding: 0 10px;
-    background-color: rgba(240, 240, 240, 1);
-    -webkit-font-smoothing: antialiased;
-  }
-
-  .input-box .input:hover {
-    border: none;
-    background-color: rgba(255, 255, 255, 1);
-  }
-
-  .input-box .input:focus {
-    border: none;
-    background-color: rgba(255, 255, 255, 1);
-  }
 </style>

cmd/openem-ingestor-app/frontend/src/List.svelte

@@ -3,7 +3,7 @@
 </script>
 
 <h3>Selected Folders:</h3>
-<div style="overflow-y: scroll; height:400px;">
+<div style="overflow-y: scroll; flex 1 1 auto;">
   <ul>
     {#each Object.entries(items) as [id, item]}
       <li>