diff --git a/apps/studio/README.md b/apps/studio/README.md index 1f48a00..9705e93 100644 --- a/apps/studio/README.md +++ b/apps/studio/README.md @@ -16,6 +16,12 @@ cmake --build build --target afs_studio ./build/apps/studio/afs_studio ``` +## Data sources + +- Training data path: `~/src/training` if present, otherwise `~/.context/training` (override with CLI arg). +- Context graph: `AFS_GRAPH_PATH` or `${AFS_CONTEXT_ROOT}/index/afs_graph.json` (defaults to `~/src/context` or `~/.context`). +- Dataset registry: `AFS_DATASET_REGISTRY` or `${AFS_TRAINING_ROOT}/index/dataset_registry.json`. + ## Features - **Dashboard**: Training metrics overview diff --git a/apps/studio/src/app.cc b/apps/studio/src/app.cc index ae8424f..ab1b4d0 100644 --- a/apps/studio/src/app.cc +++ b/apps/studio/src/app.cc @@ -211,6 +211,23 @@ void App::SyncDataBackedState() { mission.progress = 1.0f; state_.missions.push_back(std::move(mission)); } + + const auto& context_graph = loader_.GetContextGraph(); + if (!context_graph.labels.empty()) { + state_.knowledge_concepts = context_graph.labels; + state_.knowledge_nodes_x = context_graph.nodes_x; + state_.knowledge_nodes_y = context_graph.nodes_y; + state_.knowledge_edges.clear(); + state_.knowledge_edges.reserve(context_graph.edges.size()); + for (const auto& edge : context_graph.edges) { + state_.knowledge_edges.push_back({edge.from, edge.to}); + } + } else if (!loader_.GetContextGraphError().empty()) { + state_.knowledge_concepts.clear(); + state_.knowledge_nodes_x.clear(); + state_.knowledge_nodes_y.clear(); + state_.knowledge_edges.clear(); + } } void App::SeedDefaultState() { diff --git a/apps/studio/src/data_loader.cc b/apps/studio/src/data_loader.cc index 39e0de6..a686271 100644 --- a/apps/studio/src/data_loader.cc +++ b/apps/studio/src/data_loader.cc @@ -20,6 +20,7 @@ namespace { using json = nlohmann::json; constexpr size_t kTrendWindow = 5; +constexpr float kPi = 3.14159265f; std::optional ResolveHafsScawfulRoot() { const char* env_root = std::getenv("AFS_SCAWFUL_ROOT"); @@ -43,6 +44,83 @@ std::optional ResolveHafsScawfulRoot() { return std::nullopt; } +std::optional ResolveTrunkRoot() { + const char* env_root = std::getenv("TRUNK_ROOT"); + if (env_root && env_root[0] != '\0') { + auto path = studio::core::FileSystem::ResolvePath(env_root); + if (studio::core::FileSystem::Exists(path)) { + return path; + } + } + + auto path = studio::core::FileSystem::ResolvePath("~/src/trunk"); + if (studio::core::FileSystem::Exists(path)) { + return path; + } + + return std::nullopt; +} + +std::filesystem::path ResolveContextRoot() { + const char* env_root = std::getenv("AFS_CONTEXT_ROOT"); + if (env_root && env_root[0] != '\0') { + auto path = studio::core::FileSystem::ResolvePath(env_root); + if (studio::core::FileSystem::Exists(path)) { + return path; + } + } + + auto candidate = studio::core::FileSystem::ResolvePath("~/src/context"); + if (studio::core::FileSystem::Exists(candidate)) { + return candidate; + } + + auto fallback = studio::core::FileSystem::ResolvePath("~/.context"); + if (studio::core::FileSystem::Exists(fallback)) { + return fallback; + } + + return candidate; +} + +std::filesystem::path ResolveTrainingRoot() { + const char* env_root = std::getenv("AFS_TRAINING_ROOT"); + if (env_root && env_root[0] != '\0') { + auto path = studio::core::FileSystem::ResolvePath(env_root); + if (studio::core::FileSystem::Exists(path)) { + return path; + } + } + + auto candidate = studio::core::FileSystem::ResolvePath("~/src/training"); + if (studio::core::FileSystem::Exists(candidate)) { + return candidate; + } + + auto fallback = studio::core::FileSystem::ResolvePath("~/.context/training"); + if (studio::core::FileSystem::Exists(fallback)) { + return fallback; + } + + return candidate; +} + +std::filesystem::path ResolveContextGraphPath() { + const char* env_path = std::getenv("AFS_GRAPH_PATH"); + if (env_path && env_path[0] != '\0') { + return studio::core::FileSystem::ResolvePath(env_path); + } + return ResolveContextRoot() / "index" / "afs_graph.json"; +} + +std::filesystem::path ResolveDatasetRegistryPath() { + const char* env_path = std::getenv("AFS_DATASET_REGISTRY"); + if (env_path && env_path[0] != '\0') { + return studio::core::FileSystem::ResolvePath(env_path); + } + return ResolveTrainingRoot() / "index" / "dataset_registry.json"; +} + constexpr float kTrendDeltaThreshold = 0.05f; bool IsWhitespaceOnly(const std::string& s) { @@ -96,9 +174,9 @@ bool DataLoader::Refresh() { last_status_.error_count = 1; last_status_.last_error = last_error_; last_status_.last_error_source = "data_path"; - return false; + } else { + LOG_INFO("DataLoader: Refreshing from " + data_path_); } - LOG_INFO("DataLoader: Refreshing from " + data_path_); auto next_quality_trends = quality_trends_; auto next_generator_stats = generator_stats_; @@ -109,6 +187,8 @@ bool DataLoader::Refresh() { auto next_optimization_data = optimization_data_; auto next_curated_hacks = curated_hacks_; auto next_resource_index = resource_index_; + auto next_dataset_registry = dataset_registry_; + auto next_context_graph = context_graph_; LoadResult quality = LoadQualityFeedback(&next_quality_trends, &next_generator_stats, @@ -187,6 +267,28 @@ bool DataLoader::Refresh() { resource_index_ = std::move(next_resource_index); resource_index_error_.clear(); } + + LoadResult registry = LoadDatasetRegistry(&next_dataset_registry); + if (!registry.found) { + dataset_registry_ = DatasetRegistryData{}; + dataset_registry_error_ = "dataset_registry.json not found"; + } else if (!registry.ok) { + dataset_registry_error_ = registry.error; + } else { + dataset_registry_ = std::move(next_dataset_registry); + dataset_registry_error_.clear(); + } + + LoadResult context_graph = LoadContextGraph(&next_context_graph); + if (!context_graph.found) { + context_graph_ = ContextGraphData{}; + context_graph_error_ = "afs_graph.json not found"; + } else if (!context_graph.ok) { + context_graph_error_ = context_graph.error; + } else { + context_graph_ = std::move(next_context_graph); + context_graph_error_.clear(); + } // Update Mounts status mounts_.clear(); @@ -201,15 +303,25 @@ bool DataLoader::Refresh() { }; add_mount("Code", "~/Code"); + auto trunk_root = ResolveTrunkRoot(); + if (trunk_root) { + add_mount("Trunk", trunk_root->string()); + } auto scawful_root = ResolveHafsScawfulRoot(); if (scawful_root) { add_mount("afs_scawful", scawful_root->string()); } add_mount("usdasm", "~/Code/usdasm"); add_mount("Medical Mechanica (D)", "/Users/scawful/Mounts/mm-d/afs_training"); - add_mount("Oracle-of-Secrets", "~/Code/Oracle-of-Secrets"); - add_mount("yaze", "~/Code/yaze"); - add_mount("System Context", "~/.context"); + if (trunk_root) { + add_mount("Oracle-of-Secrets", (trunk_root.value() / "scawful/retro/oracle-of-secrets").string()); + add_mount("yaze", (trunk_root.value() / "scawful/retro/yaze").string()); + } else { + add_mount("Oracle-of-Secrets", "~/Code/Oracle-of-Secrets"); + add_mount("yaze", "~/Code/yaze"); + } + add_mount("AFS Context", ResolveContextRoot().string()); + add_mount("AFS Training", ResolveTrainingRoot().string()); has_data_ = !quality_trends_.empty() || !generator_stats_.empty() || !embedding_regions_.empty() || !training_runs_.empty() || @@ -585,6 +697,150 @@ DataLoader::LoadResult DataLoader::LoadResourceIndex(ResourceIndexData* resource return result; } +DataLoader::LoadResult DataLoader::LoadDatasetRegistry(DatasetRegistryData* dataset_registry) { + LoadResult result; + std::filesystem::path path = ResolveDatasetRegistryPath(); + if (!path_exists_(path.string())) { + return result; + } + result.found = true; + LOG_INFO("DataLoader: Loading " + path.string()); + + std::string content; + std::string read_error; + if (!file_reader_(path.string(), &content, &read_error) || content.empty() || + IsWhitespaceOnly(content)) { + result.ok = false; + result.error = read_error.empty() ? "dataset_registry.json is empty" : read_error; + return result; + } + + try { + json data = json::parse(content); + dataset_registry->generated_at = data.value("generated_at", ""); + dataset_registry->datasets.clear(); + + if (!data.contains("datasets") || !data["datasets"].is_array()) { + result.ok = false; + result.error = "dataset_registry.json missing datasets array"; + return result; + } + + for (const auto& entry : data["datasets"]) { + DatasetEntry dataset; + dataset.name = entry.value("name", ""); + dataset.path = entry.value("path", ""); + dataset.size_bytes = static_cast(entry.value("size_bytes", 0)); + dataset.updated_at = entry.value("updated_at", ""); + if (entry.contains("files") && entry["files"].is_array()) { + for (const auto& file : entry["files"]) { + if (file.is_string()) { + dataset.files.push_back(file.get()); + } + } + } + dataset_registry->datasets.push_back(std::move(dataset)); + } + + result.ok = true; + } catch (const std::exception& e) { + result.ok = false; + result.error = std::string("Failed to parse dataset_registry.json: ") + e.what(); + } + + return result; +} + +DataLoader::LoadResult DataLoader::LoadContextGraph(ContextGraphData* context_graph) { + LoadResult result; + std::filesystem::path path = ResolveContextGraphPath(); + if (!path_exists_(path.string())) { + return result; + } + result.found = true; + LOG_INFO("DataLoader: Loading " + path.string()); + + std::string content; + std::string read_error; + if (!file_reader_(path.string(), &content, &read_error) || content.empty() || + IsWhitespaceOnly(content)) { + result.ok = false; + result.error = read_error.empty() ? "afs_graph.json is empty" : read_error; + return result; + } + + try { + json data = json::parse(content); + if (!data.contains("contexts") || !data["contexts"].is_array()) { + result.ok = false; + result.error = "afs_graph.json missing contexts array"; + return result; + } + + context_graph->labels.clear(); + context_graph->nodes_x.clear(); + context_graph->nodes_y.clear(); + context_graph->edges.clear(); + context_graph->context_count = 0; + context_graph->mount_count = 0; + context_graph->source_path = path.string(); + + const auto& contexts = data["contexts"]; + const size_t context_total = contexts.size(); + context_graph->context_count = static_cast(context_total); + + for (size_t i = 0; i < context_total; ++i) { + const auto& ctx = contexts[i]; + std::string name = ctx.value("name", "context"); + float angle = (context_total > 0) + ? (2.0f * kPi * static_cast(i) / static_cast(context_total)) + : 0.0f; + float cx = std::cos(angle); + float cy = std::sin(angle); + + int ctx_index = static_cast(context_graph->labels.size()); + context_graph->labels.push_back(name); + context_graph->nodes_x.push_back(cx); + context_graph->nodes_y.push_back(cy); + + if (!ctx.contains("mounts") || !ctx["mounts"].is_array()) { + continue; + } + + const auto& mounts = ctx["mounts"]; + const size_t mount_total = mounts.size(); + if (mount_total == 0) { + continue; + } + + float ring = 0.35f + 0.02f * static_cast(mount_total); + for (size_t j = 0; j < mount_total; ++j) { + const auto& mount = mounts[j]; + std::string mount_name = mount.value("name", "mount"); + std::string mount_type = mount.value("mount_type", ""); + std::string label = mount_type.empty() ? mount_name : (mount_type + ":" + mount_name); + float local_angle = (2.0f * kPi * static_cast(j) / static_cast(mount_total)); + float mx = cx + std::cos(local_angle) * ring; + float my = cy + std::sin(local_angle) * ring; + + int mount_index = static_cast(context_graph->labels.size()); + context_graph->labels.push_back(label); + context_graph->nodes_x.push_back(mx); + context_graph->nodes_y.push_back(my); + context_graph->edges.push_back({ctx_index, mount_index}); + context_graph->mount_count += 1; + } + } + + result.ok = true; + } catch (const std::exception& e) { + result.ok = false; + result.error = std::string("Failed to parse afs_graph.json: ") + e.what(); + } + + return result; +} + void DataLoader::MountDrive(const std::string& name) { auto scawful_root = ResolveHafsScawfulRoot(); std::filesystem::path script_path = scawful_root diff --git a/apps/studio/src/data_loader.h b/apps/studio/src/data_loader.h index 6c57345..96b66ce 100644 --- a/apps/studio/src/data_loader.h +++ b/apps/studio/src/data_loader.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -96,6 +97,38 @@ struct ResourceIndexData { std::map by_type; }; +/// Dataset registry entry. +struct DatasetEntry { + std::string name; + std::string path; + std::uint64_t size_bytes = 0; + std::string updated_at; + std::vector files; +}; + +/// Dataset registry summary. +struct DatasetRegistryData { + std::string generated_at; + std::vector datasets; +}; + +/// Context graph edge. +struct ContextGraphEdge { + int from = -1; + int to = -1; +}; + +/// Context graph data (contexts + mounts). +struct ContextGraphData { + std::string source_path; + int context_count = 0; + int mount_count = 0; + std::vector labels; + std::vector nodes_x; + std::vector nodes_y; + std::vector edges; +}; + /// Optimization metrics. struct OptimizationData { std::map domain_effectiveness; @@ -175,6 +208,10 @@ class DataLoader { } const ResourceIndexData& GetResourceIndex() const { return resource_index_; } const std::string& GetResourceIndexError() const { return resource_index_error_; } + const DatasetRegistryData& GetDatasetRegistry() const { return dataset_registry_; } + const std::string& GetDatasetRegistryError() const { return dataset_registry_error_; } + const ContextGraphData& GetContextGraph() const { return context_graph_; } + const std::string& GetContextGraphError() const { return context_graph_error_; } const std::vector& GetMounts() const { return mounts_; } @@ -204,6 +241,8 @@ class DataLoader { OptimizationData* optimization_data); LoadResult LoadCuratedHacks(std::vector* curated_hacks); LoadResult LoadResourceIndex(ResourceIndexData* resource_index); + LoadResult LoadDatasetRegistry(DatasetRegistryData* dataset_registry); + LoadResult LoadContextGraph(ContextGraphData* context_graph); std::string data_path_; FileReader file_reader_; @@ -223,6 +262,10 @@ class DataLoader { std::string curated_hacks_error_; ResourceIndexData resource_index_; std::string resource_index_error_; + DatasetRegistryData dataset_registry_; + std::string dataset_registry_error_; + ContextGraphData context_graph_; + std::string context_graph_error_; std::vector mounts_; std::map domain_visibility_; }; diff --git a/apps/studio/src/main.cc b/apps/studio/src/main.cc index e8f1fa5..9587c96 100644 --- a/apps/studio/src/main.cc +++ b/apps/studio/src/main.cc @@ -1,7 +1,7 @@ /// AFS Training Data Visualization - Main Entry Point /// /// Usage: afs_viz [data_path] -/// data_path: Path to training data directory (default: ~/.context/training) +/// data_path: Path to training data directory (default: ~/src/training if present) /// /// Build: /// cmake -B build -S src/cc -DAFS_BUILD_VIZ=ON @@ -27,7 +27,8 @@ int main(int argc, char* argv[]) { if (argc > 1) { data_path_str = argv[1]; } else { - data_path_str = "~/.context/training"; + auto preferred = FileSystem::ResolvePath("~/src/training"); + data_path_str = FileSystem::Exists(preferred) ? preferred.string() : "~/.context/training"; } std::filesystem::path data_path = FileSystem::ResolvePath(data_path_str); diff --git a/apps/studio/src/models/state.h b/apps/studio/src/models/state.h index fdf1150..0f0bd5a 100644 --- a/apps/studio/src/models/state.h +++ b/apps/studio/src/models/state.h @@ -245,7 +245,7 @@ struct AppState { bool is_rendering_expanded_plot = false; std::vector custom_grid_slots; - // Knowledge Graph + // Context Graph std::vector knowledge_concepts; std::vector knowledge_nodes_x; std::vector knowledge_nodes_y; diff --git a/apps/studio/src/ui/components/charts.cc b/apps/studio/src/ui/components/charts.cc index 70dd137..0394c68 100644 --- a/apps/studio/src/ui/components/charts.cc +++ b/apps/studio/src/ui/components/charts.cc @@ -1146,15 +1146,24 @@ void RenderMountsChart(AppState& state, const DataLoader& loader) { void RenderKnowledgeGraphChart(AppState& state, const DataLoader& loader) { RenderChartHeader(PlotKind::KnowledgeGraph, - "KNOWLEDGE GRAPH", - "A topological view of semantic relationships between concepts within the training corpus.", + "CONTEXT GRAPH", + "AFS context map showing workspace and mount relationships.", state); if (state.knowledge_concepts.empty()) { - ImGui::TextDisabled("No knowledge graph data available"); + const auto& error = loader.GetContextGraphError(); + if (!error.empty()) { + ImGui::TextColored(ImVec4(0.9f, 0.5f, 0.2f, 1.0f), "%s", error.c_str()); + } + ImGui::TextDisabled("No context graph data available"); return; } + const auto& graph = loader.GetContextGraph(); + if (graph.context_count > 0 || graph.mount_count > 0) { + ImGui::TextDisabled("Contexts: %d Mounts: %d", graph.context_count, graph.mount_count); + } + ImPlotFlags plot_flags = BasePlotFlags(state, false); ApplyPremiumPlotStyles("##KnowledgeGraph", state); if (ImPlot::BeginPlot("##KnowledgeGraph", ImGui::GetContentRegionAvail(), plot_flags)) { diff --git a/apps/studio/src/ui/components/companion_panels.cc b/apps/studio/src/ui/components/companion_panels.cc index 585e114..1aacd8d 100644 --- a/apps/studio/src/ui/components/companion_panels.cc +++ b/apps/studio/src/ui/components/companion_panels.cc @@ -257,6 +257,7 @@ void CompanionPanels::RenderInspectorPanel(AppState& state, const DataLoader& lo case PlotKind::EvalMetrics: graph_name = "Eval Metrics"; break; case PlotKind::AgentUtilization: graph_name = "Agent Utilization"; break; case PlotKind::MountsStatus: graph_name = "Mounts Status"; break; + case PlotKind::KnowledgeGraph: graph_name = "Context Graph"; break; default: break; } ImGui::Text("Graph: %s", graph_name); diff --git a/apps/studio/src/ui/components/graph_browser.cc b/apps/studio/src/ui/components/graph_browser.cc index 29095c6..ba91404 100644 --- a/apps/studio/src/ui/components/graph_browser.cc +++ b/apps/studio/src/ui/components/graph_browser.cc @@ -36,7 +36,7 @@ void GraphBrowser::InitializeGraphRegistry() { // Embedding Category {PlotKind::EmbeddingDensity, "Embedding Density", "Embedding space density visualization", GraphCategory::Embedding, false, true, true, true}, {PlotKind::LatentSpace, "Latent Space", "2D latent space projection", GraphCategory::Embedding, false, true, true, true}, - {PlotKind::KnowledgeGraph, "Knowledge Graph", "Knowledge concept relationships", GraphCategory::Embedding, false, false, false, false}, + {PlotKind::KnowledgeGraph, "Context Graph", "AFS context and mount relationships", GraphCategory::Embedding, false, false, false, false}, // Optimization Category {PlotKind::GeneratorEfficiency, "Generator Efficiency", "Generator performance metrics", GraphCategory::Optimization, true, true, true, true}, diff --git a/apps/studio/src/ui/components/panels.cc b/apps/studio/src/ui/components/panels.cc index f280283..1a04724 100644 --- a/apps/studio/src/ui/components/panels.cc +++ b/apps/studio/src/ui/components/panels.cc @@ -501,6 +501,8 @@ void RenderDatasetPanel(AppState& state, const DataLoader& loader) { const auto& runs = loader.GetTrainingRuns(); const auto& generators = loader.GetGeneratorStats(); const auto& coverage = loader.GetCoverage(); + const auto& dataset_registry = loader.GetDatasetRegistry(); + const auto& dataset_error = loader.GetDatasetRegistryError(); if (ImGui::BeginTabBar("DatasetTabs")) { if (ImGui::BeginTabItem("Training Runs")) { @@ -653,6 +655,51 @@ void RenderDatasetPanel(AppState& state, const DataLoader& loader) { ImGui::EndTabItem(); } + if (ImGui::BeginTabItem("Datasets")) { + if (!dataset_error.empty()) { + ImGui::TextColored(ImVec4(0.9f, 0.5f, 0.2f, 1.0f), "%s", dataset_error.c_str()); + } + + if (dataset_registry.datasets.empty()) { + ImGui::TextDisabled("No dataset registry loaded."); + } else { + if (!dataset_registry.generated_at.empty()) { + ImGui::TextDisabled("Generated at: %s", dataset_registry.generated_at.c_str()); + } + + if (ImGui::BeginTable("DatasetRegistryTable", 4, + ImGuiTableFlags_RowBg | ImGuiTableFlags_Resizable | + ImGuiTableFlags_Borders | ImGuiTableFlags_ScrollY | ImGuiTableFlags_SizingStretchProp)) { + ImGui::TableSetupColumn("Dataset", ImGuiTableColumnFlags_WidthStretch, 1.4f); + ImGui::TableSetupColumn("Files", ImGuiTableColumnFlags_WidthFixed, 80); + ImGui::TableSetupColumn("Size (MB)", ImGuiTableColumnFlags_WidthFixed, 100); + ImGui::TableSetupColumn("Updated", ImGuiTableColumnFlags_WidthStretch, 1.1f); + ImGui::TableHeadersRow(); + + for (const auto& dataset : dataset_registry.datasets) { + ImGui::TableNextRow(); + ImGui::TableNextColumn(); + ImGui::Text("%s", dataset.name.c_str()); + if (ImGui::IsItemHovered() && !dataset.path.empty()) { + ImGui::BeginTooltip(); + ImGui::Text("%s", dataset.path.c_str()); + ImGui::EndTooltip(); + } + ImGui::TableNextColumn(); + ImGui::Text("%zu", dataset.files.size()); + ImGui::TableNextColumn(); + double size_mb = static_cast(dataset.size_bytes) / (1024.0 * 1024.0); + ImGui::Text("%.2f", size_mb); + ImGui::TableNextColumn(); + ImGui::Text("%s", dataset.updated_at.empty() ? "-" : dataset.updated_at.c_str()); + } + ImGui::EndTable(); + } + } + + ImGui::EndTabItem(); + } + if (ImGui::BeginTabItem("Generators")) { ImGui::InputTextWithHint("##GenFilter", "Filter by generator name", state.generator_filter.data(), state.generator_filter.size()); ImGui::SameLine(); diff --git a/apps/studio/src/ui/core.cc b/apps/studio/src/ui/core.cc index 27380d0..25eccc9 100644 --- a/apps/studio/src/ui/core.cc +++ b/apps/studio/src/ui/core.cc @@ -243,7 +243,7 @@ const std::vector& PlotOptions() { {PlotKind::MissionProgress, "Mission Progress"}, {PlotKind::EvalMetrics, "Eval Metrics"}, {PlotKind::Rejections, "Rejection Reasons"}, - {PlotKind::KnowledgeGraph, "Knowledge Graph"}, + {PlotKind::KnowledgeGraph, "Context Graph"}, {PlotKind::LatentSpace, "Latent Space"}, {PlotKind::Effectiveness, "Domain Effectiveness"}, {PlotKind::Thresholds, "Threshold Sensitivity"},