Merge pull request #79 from elpassion/sections

Sections
elpassion · Oct 23, 2024 · d8b6d96 · d8b6d96
2 parents 6256451 + 4b0f2e8
commit d8b6d96
Show file tree

Hide file tree

Showing 27 changed files with 1,678 additions and 261 deletions.
diff --git a/apps/api/lib/buildel/blocks/chat.ex b/apps/api/lib/buildel/blocks/chat.ex
@@ -43,12 +43,8 @@ defmodule Buildel.Blocks.Chat do
           options_schema(%{
             "required" => [
               "description",
-              "api_key",
-              "api_type",
-              "model",
-              "endpoint",
+              "model_section",
               "chat_memory_type",
-              "temperature",
               "system_message",
               "messages",
               "prompt_template"
@@ -68,66 +64,88 @@ defmodule Buildel.Blocks.Chat do
                     }
                   }
                 },
-                api_type: %{
-                  "type" => "string",
-                  "title" => "Model API type",
-                  "description" => "The API type to use for the chat.",
-                  "enum" => ["openai", "azure", "google", "mistral", "anthropic"],
-                  "enumPresentAs" => "radio",
-                  "default" => "openai",
-                  "readonly" => true
-                },
-                api_key:
-                  secret_schema(%{
-                    "title" => "API key",
-                    "description" => "API key to use for the chat.",
-                    "descriptionWhen" => %{
-                      "opts.api_type" => %{
-                        "openai" =>
-                          "[OpenAI API key](https://platform.openai.com/api-keys) to use for the chat.",
-                        "azure" => "Azure API key to use for the chat.",
-                        "google" => "Google API key to use for the chat.",
-                        "mistral" =>
-                          "[Mistral API key](https://console.mistral.ai/api-keys/) to use for the chat.",
-                        "anthropic" =>
-                          "[Anthropic API key](https://www.anthropic.com/api) to use for the chat."
-                      }
-                    },
-                    "defaultWhen" => %{
-                      "opts.api_type" => %{
-                        "openai" => "__openai",
-                        "azure" => "__azure",
-                        "google" => "__google",
-                        "mistral" => "__mistral",
-                        "anthropic" => "__anthropic"
-                      }
-                    }
-                  }),
-                endpoint: %{
-                  "type" => "string",
-                  "title" => "Endpoint",
-                  "description" => "The endpoint to use for the chat.",
-                  "defaultWhen" => %{
-                    "opts.api_type" => %{
-                      "openai" => "https://api.openai.com/v1",
-                      "azure" =>
-                        "https://{resource_name}.openai.azure.com/openai/deployments/{deployment_name}",
-                      "google" => "https://generativelanguage.googleapis.com/v1beta/models",
-                      "mistral" => "https://api.mistral.ai/v1",
-                      "anthropic" => "https://api.anthropic.com/v1"
-                    }
-                  },
-                  "minLength" => 1
-                },
-                model: %{
-                  "type" => "string",
+                model_section: %{
+                  "type" => "section",
                   "title" => "Model",
                   "description" => "The model to use for the chat.",
-                  "url" =>
-                    "/api/organizations/{{organization_id}}/models?api_type={{opts.api_type}}&endpoint={{opts.endpoint}}&api_key={{opts.api_key}}",
-                  "presentAs" => "async-select",
-                  "minLength" => 1,
-                  "readonly" => true
+                  "required" => ["api_type", "api_key", "endpoint", "model", "temperature"],
+                  "properties" =>
+                    Jason.OrderedObject.new(
+                      api_type: %{
+                        "type" => "string",
+                        "title" => "Model API type",
+                        "description" => "The API type to use for the chat.",
+                        "enum" => ["openai", "azure", "google", "mistral", "anthropic"],
+                        "enumPresentAs" => "radio",
+                        "default" => "openai",
+                        "readonly" => true
+                      },
+                      api_key:
+                        secret_schema(%{
+                          "title" => "API key",
+                          "description" => "API key to use for the chat.",
+                          "descriptionWhen" => %{
+                            "opts.model_section.api_type" => %{
+                              "openai" =>
+                                "[OpenAI API key](https://platform.openai.com/api-keys) to use for the chat.",
+                              "azure" => "Azure API key to use for the chat.",
+                              "google" => "Google API key to use for the chat.",
+                              "mistral" =>
+                                "[Mistral API key](https://console.mistral.ai/api-keys/) to use for the chat.",
+                              "anthropic" =>
+                                "[Anthropic API key](https://www.anthropic.com/api) to use for the chat."
+                            }
+                          },
+                          "defaultWhen" => %{
+                            "opts.model_section.api_type" => %{
+                              "openai" => "__openai",
+                              "azure" => "__azure",
+                              "google" => "__google",
+                              "mistral" => "__mistral",
+                              "anthropic" => "__anthropic"
+                            }
+                          }
+                        }),
+                      endpoint: %{
+                        "errorMessages" => %{
+                          "minLength" => "Endpoint is required."
+                        },
+                        "type" => "string",
+                        "title" => "Endpoint",
+                        "description" => "The endpoint to use for the chat.",
+                        "defaultWhen" => %{
+                          "opts.model_section.api_type" => %{
+                            "openai" => "https://api.openai.com/v1",
+                            "azure" =>
+                              "https://{resource_name}.openai.azure.com/openai/deployments/{deployment_name}",
+                            "google" => "https://generativelanguage.googleapis.com/v1beta/models",
+                            "mistral" => "https://api.mistral.ai/v1",
+                            "anthropic" => "https://api.anthropic.com/v1"
+                          }
+                        },
+                        "minLength" => 1
+                      },
+                      model: %{
+                        "type" => "string",
+                        "title" => "Model",
+                        "description" => "The model to use for the chat.",
+                        "url" =>
+                          "/api/organizations/{{organization_id}}/models?api_type={{opts.model_section.api_type}}&endpoint={{opts.model_section.endpoint}}&api_key={{opts.model_section.api_key}}",
+                        "presentAs" => "async-select",
+                        "minLength" => 1,
+                        "readonly" => true
+                      },
+                      temperature: %{
+                        "type" => "number",
+                        "title" => "Temperature",
+                        "description" => "The temperature of the chat.",
+                        "default" => 0.7,
+                        "minimum" => 0.0,
+                        "maximum" => 2.0,
+                        "step" => 0.1,
+                        "readonly" => true
+                      }
+                    )
                 },
                 chat_memory_type: %{
                   "type" => "string",
@@ -138,16 +156,6 @@ defmodule Buildel.Blocks.Chat do
                   "default" => "full",
                   "minLength" => 1
                 },
-                temperature: %{
-                  "type" => "number",
-                  "title" => "Temperature",
-                  "description" => "The temperature of the chat.",
-                  "default" => 0.7,
-                  "minimum" => 0.0,
-                  "maximum" => 2.0,
-                  "step" => 0.1,
-                  "readonly" => true
-                },
                 max_tokens: %{
                   "type" => "number",
                   "title" => "Maximum tokens",

diff --git a/apps/api/lib/buildel/migrations/add_sections_to_blocks.ex b/apps/api/lib/buildel/migrations/add_sections_to_blocks.ex
@@ -0,0 +1,65 @@
+defmodule Buildel.Migrations.AddSectionsToChatBlocks do
+  alias Buildel.PipelineConfigMigrator
+
+  def migrate(pipelines, aliases) do
+    pipelines =
+      pipelines
+      |> Enum.map(fn pipeline ->
+        move_chat_opts_to_model_section(pipeline)
+      end)
+
+    aliases =
+      aliases
+      |> Enum.map(fn pipeline_alias ->
+        move_chat_opts_to_model_section(pipeline_alias)
+      end)
+
+    %{pipelines: pipelines, aliases: aliases}
+  end
+
+  def rollback(pipelines, aliases) do
+    pipelines =
+      pipelines
+      |> Enum.map(fn pipeline ->
+        move_chat_opts_from_model_section(pipeline)
+      end)
+
+    aliases =
+      aliases
+      |> Enum.map(fn pipeline_alias ->
+        move_chat_opts_from_model_section(pipeline_alias)
+      end)
+
+    %{pipelines: pipelines, aliases: aliases}
+  end
+
+  defp move_chat_opts_to_model_section(%{config: config} = pipeline_or_alias) do
+    %{
+      pipeline_or_alias
+      | config:
+          config
+          |> PipelineConfigMigrator.move_opts_to_section("chat", "model_section", [
+            "api_type",
+            "api_key",
+            "model",
+            "endpoint",
+            "temperature"
+          ])
+    }
+  end
+
+  defp move_chat_opts_from_model_section(%{config: config} = pipeline_or_alias) do
+    %{
+      pipeline_or_alias
+      | config:
+          config
+          |> PipelineConfigMigrator.move_opts_from_section("chat", "model_section", [
+            "api_type",
+            "api_key",
+            "model",
+            "endpoint",
+            "temperature"
+          ])
+    }
+  end
+end
diff --git a/apps/api/lib/buildel/pipeline_config_migrator.ex b/apps/api/lib/buildel/pipeline_config_migrator.ex
@@ -1,35 +1,35 @@
 defmodule Buildel.PipelineConfigMigrator do
+  alias Buildel.BlockConfigMigrator
+
   def rename_block_types(
         %{"blocks" => blocks} = config,
         block_type,
         new_block_type
       ) do
     %{
       config
-      | "blocks" =>
-          Buildel.BlockConfigMigrator.rename_block_type(blocks, block_type, new_block_type)
+      | "blocks" => BlockConfigMigrator.rename_block_type(blocks, block_type, new_block_type)
     }
   end
 
   def add_block_opt(%{"blocks" => blocks} = config, block_type, opt, value) do
     %{
       config
-      | "blocks" => Buildel.BlockConfigMigrator.add_block_opt(blocks, block_type, opt, value)
+      | "blocks" => BlockConfigMigrator.add_block_opt(blocks, block_type, opt, value)
     }
   end
 
   def remove_block_opt(%{"blocks" => blocks} = config, block_type, opt) do
     %{
       config
-      | "blocks" => Buildel.BlockConfigMigrator.remove_block_opt(blocks, block_type, opt)
+      | "blocks" => BlockConfigMigrator.remove_block_opt(blocks, block_type, opt)
     }
   end
 
   def update_block_config(%{"blocks" => blocks} = config, block_type, new_config) do
     %{
       config
-      | "blocks" =>
-          Buildel.BlockConfigMigrator.update_block_config(blocks, block_type, new_config)
+      | "blocks" => BlockConfigMigrator.update_block_config(blocks, block_type, new_config)
     }
   end
 
@@ -94,4 +94,27 @@ defmodule Buildel.PipelineConfigMigrator do
             ]
     }
   end
+
+  def move_opts_to_section(config, block_type, section_name, opts) do
+    config
+    |> update_block_config(block_type, fn config ->
+      section_opts = opts |> Enum.map(&{&1, config["opts"][&1]}) |> Enum.into(%{})
+
+      config = BlockConfigMigrator.add_block_opt(config, section_name, section_opts)
+
+      opts |> Enum.reduce(config, &BlockConfigMigrator.remove_block_opt(&2, &1))
+    end)
+  end
+
+  def move_opts_from_section(config, block_type, section_name, opts) do
+    config
+    |> update_block_config(block_type, fn config ->
+      section_opts = opts |> Enum.map(&{&1, config["opts"][section_name][&1]})
+
+      config = BlockConfigMigrator.remove_block_opt(config, section_name)
+
+      section_opts
+      |> Enum.reduce(config, &BlockConfigMigrator.add_block_opt(&2, elem(&1, 0), elem(&1, 1)))
+    end)
+  end
 end
diff --git a/apps/api/lib/buildel/pipelines.ex b/apps/api/lib/buildel/pipelines.ex
@@ -356,6 +356,7 @@ defmodule Buildel.Pipelines do
         opts:
           block["opts"]
           |> keys_to_atoms()
+          |> flatten_map()
           |> Map.put(:metadata, config["metadata"] || %{})
       }
     end)
@@ -405,6 +406,20 @@ defmodule Buildel.Pipelines do
     end
   end
 
+  defp flatten_map(map), do: flatten_map(map, %{})
+
+  defp flatten_map(%{} = map, acc) do
+    map
+    |> Enum.reduce(acc, fn {key, value}, acc ->
+      case value do
+        %{} -> flatten_map(value, acc)
+        _ -> Map.put(acc, key, value)
+      end
+    end)
+  end
+
+  defp flatten_map(_value, acc), do: acc
+
   defp block_valid?(config, block_config) do
     cond do
       config["blocks"] |> Enum.any?(&(&1["name"] == block_config.name)) ->
@@ -454,7 +469,10 @@ defmodule Buildel.Pipelines do
                   "block_name" => connection.from.block_name,
                   "output_name" => connection.from.name
                 },
-                "opts" => %{"reset" => connection.opts.reset, "optional" => connection.opts.optional},
+                "opts" => %{
+                  "reset" => connection.opts.reset,
+                  "optional" => connection.opts.optional
+                },
                 "to" => %{
                   "block_name" => connection.to.block_name,
                   "input_name" => connection.to.name
@@ -564,6 +582,10 @@ defmodule Buildel.Pipelines do
     Repo.delete(alias)
   end
 
+  def list_aliases() do
+    Repo.all(Alias)
+  end
+
   defp keys_to_atoms(string_key_map) when is_map(string_key_map) do
     for {key, val} <- string_key_map,
         into: %{},

diff --git a/apps/api/lib/buildel/run_logs.ex b/apps/api/lib/buildel/run_logs.ex
@@ -68,7 +68,7 @@ defmodule Buildel.RunLogs do
       limit: attrs.limit,
       include_total_count: true,
       cursor_fields: [{:inserted_at, :desc}, {:id, :asc}],
-      after: attrs.after,
+      after: attrs.after
     )
   end
 

diff --git a/apps/api/lib/buildel_web/controllers/experiments/experiment_json.ex b/apps/api/lib/buildel_web/controllers/experiments/experiment_json.ex
@@ -27,7 +27,7 @@ defmodule BuildelWeb.ExperimentJSON do
     nil
   end
 
-  defp map_pipeline(%{"id": id, "name": name}) do
+  defp map_pipeline(%{"id" => id, "name" => name}) do
     %{
       id: id,
       name: name
-Original file line number
+Diff line change
@@ Expand Up / @@ -27,7 +27,7 @@ defmodule BuildelWeb.ExperimentJSON do @@
         nil
       end
-      defp map_pipeline(%{"id": id, "name": name}) do
+      defp map_pipeline(%{"id" => id, "name" => name}) do
         %{
           id: id,
           name: name
@@ Expand Down @@