From 7e2a7714221fec9e66042b5206f002b480714ada Mon Sep 17 00:00:00 2001 From: Nick Chase Date: Wed, 28 May 2025 11:22:30 -0400 Subject: [PATCH 1/3] Initial Agent commit --- agent/prompt.md | 92 ++++++++++++++++++++ agent/prompt.md:Zone.Identifier | 0 agent/results/test1.yaml | 104 +++++++++++++++++++++++ agent/results/test1.yaml:Zone.Identifier | 0 4 files changed, 196 insertions(+) create mode 100644 agent/prompt.md create mode 100644 agent/prompt.md:Zone.Identifier create mode 100644 agent/results/test1.yaml create mode 100644 agent/results/test1.yaml:Zone.Identifier diff --git a/agent/prompt.md b/agent/prompt.md new file mode 100644 index 0000000..b961cd0 --- /dev/null +++ b/agent/prompt.md @@ -0,0 +1,92 @@ +# Introduction and instructions + +*Current version: 0.1 (Please increment when making changes)* + +We are creating an agent that will update the Model Openness Tool. +We're starting by running the prompt manually to make sure that it works. + +To test, run these prompts in order: + +1. Setup prompt +2. Instructions prompt +3. Model request prompt + +## Setup prompt + +```yaml +I'm trying to update information about various open source LLMs for the Model Openness Tool. Read this page, including the tooltips: https://mot-dev.isitopen.ai/model/evaluate + +This page ultimately populates a YAML document, such as: + +framework: + name: 'Model Openness Framework' + version: '1.0' + date: '2024-12-15' +release: + name: DeepSeek-R1 + version: 671B + date: '2025-01-20' + license: { } + type: language + architecture: '' + origin: '' + producer: 'DeepSeek AI' + contact: '' + repository: 'https://github.com/deepseek-ai/DeepSeek-R1' + huggingface: 'https://huggingface.co/deepseek-ai/DeepSeek-R1' + components: + - + name: 'Model architecture' + description: "Well commented code for the model's architecture" + license: MIT + - + name: 'Inference code' + description: 'Code used for running the model to make predictions' + license: MIT + - + name: 'Supporting libraries and tools' + description: "Libraries and tools used in the model's development" + license: MIT + - + name: 'Model parameters (Final)' + description: 'Trained model parameters, weights and biases' + license: MIT + - + name: 'Evaluation data' + description: 'Data used for evaluating the model' + license: unlicensed + - + name: 'Model metadata' + description: 'Any model metadata including training configuration and optimizer states' + license: MIT + - + name: 'Model card' + description: 'Model details including performance metrics, intended use, and limitations' + license: unlicensed + - + name: 'Technical report' + description: 'Technical report detailing capabilities and usage instructions for the model' + license: unlicensed + - + name: 'Research paper' + description: 'Research paper detailing the development and capabilities of the model' + license: 'arXiv.org perpetual non-exclusive license 1.0' + - + name: 'Evaluation results' + description: 'The results from evaluating the model' + license: unlicensed + +Analyze the form and correlate the YAML properties with the form fields. +``` + +## Instructions prompt + +```yaml +Excellent. Now I want you to analyze an open source model. Do any research you have to answer all of these questions about the model and fill out the YAML document. If you don't know the answer, look it up online. DO NOT MAKE ANYTHING UP. Output the full YAML document, then follow up with an explanation of where you found each piece of information. This task is very important to my career. + +Do you understand? +``` + +## Model request prompt + +Here just give the LLM the name of a model, such as `DeepSeek-R1` and it will output the YAML document. diff --git a/agent/prompt.md:Zone.Identifier b/agent/prompt.md:Zone.Identifier new file mode 100644 index 0000000..e69de29 diff --git a/agent/results/test1.yaml b/agent/results/test1.yaml new file mode 100644 index 0000000..bc0cde9 --- /dev/null +++ b/agent/results/test1.yaml @@ -0,0 +1,104 @@ +run-number: 1 +tester: Nick Chase +prompt-version: 0.1 +model-used: OpenAI ChatGPT 04-mini +model-analyzed: DeepSeek-V3 +results-md: | + + ```yaml + framework: + name: "Model Openness Framework" + version: "1.0" + date: "2025-05-25" + + release: + name: "DeepSeek-V3" + version: "671B" + date: "2024-12-27" + license: "MIT" + type: "language" + architecture: "Mixture-of-Experts (MoE) with Multi-head Latent Attention (MLA) and DeepSeekMoE" + origin: "research" + producer: "DeepSeek AI" + contact: "service@deepseek.com" + repository: "https://github.com/deepseek-ai/DeepSeek-V3" + huggingface: "https://huggingface.co/deepseek-ai/DeepSeek-V3" + components: + - + name: "Model architecture" + description: "Well commented code for the model’s architecture including MLA and DeepSeekMoE modules" + license: "MIT" + - + name: "Inference code" + description: "Code used for running the model to make predictions" + license: "MIT" + - + name: "Supporting libraries and tools" + description: "Libraries and tools used for inference and deployment (e.g., SGLang, LMDeploy, TensorRT-LLM, vLLM, LightLLM)" + license: "MIT" + - + name: "Model parameters (Final)" + description: "Trained model parameters (weights and biases) for DeepSeek-V3" + license: "LICENSE-MODEL" + - + name: "Evaluation data" + description: "Data used for evaluating the model across benchmarks (e.g., MMLU, BBH, etc.)" + license: "unlicensed" + - + name: "Model metadata" + description: "Training configuration, optimizer state, and multi-token prediction module weights" + license: "MIT" + - + name: "Model card" + description: "High-level details including capabilities, limitations, and intended uses" + license: "unlicensed" + - + name: "Technical report" + description: "DeepSeek-V3 technical report published on arXiv" + license: "arXiv.org perpetual non-exclusive license 1.0" + - + name: "Research paper" + description: "Publication detailing the development of DeepSeek-V3" + license: "arXiv.org perpetual non-exclusive license 1.0" + - + name: "Evaluation results" + description: "Benchmark results and performance metrics" + license: "unlicensed" + ``` + + **Sources and Mapping:** + + - **framework.date** (the date of filling the form): set to today’s date, May 25 2025. + + - **release.name** “DeepSeek-V3” and **version** “671B” come from the GitHub model summary, which states 671B total parameters [GitHub](https://github.com/deepseek-ai/DeepSeek-V3). + + - **release.date** “2024-12-27” is the publication date of the DeepSeek-V3 technical report on arXiv [arXiv](https://arxiv.org/abs/2412.19437?utm_source=chatgpt.com). + + - **release.license** “MIT” is pulled from the repository’s code license listing on GitHub [GitHub](https://github.com/deepseek-ai/DeepSeek-V3). + + - **release.type** “language” is the model’s category as an LLM. + + - **release.architecture** is described in the repo introduction, specifying MoE with MLA and DeepSeekMoE [GitHub](https://github.com/deepseek-ai/DeepSeek-V3). + + - **release.origin** is marked “research,” reflecting that DeepSeek-V3 was developed by DeepSeek AI’s research team. + + - **release.producer** “DeepSeek AI” is the GitHub organization and company behind the model [GitHub](https://github.com/deepseek-ai/DeepSeek-V3). + + - **release.contact** “service@deepseek.com” is cited from the Hugging Face model page’s contact section for DeepSeek-V3-0324 [Hugging Face](https://huggingface.co/deepseek-ai/DeepSeek-V3-0324?utm_source=chatgpt.com). + + - **release.repository** is the GitHub URL for DeepSeek-V3 [GitHub](https://github.com/deepseek-ai/DeepSeek-V3). + + - **release.huggingface** is taken from the GitHub downloads table linking to Hugging Face [GitHub](https://github.com/deepseek-ai/DeepSeek-V3). + + - **Components & licenses**: + + - Code components (architecture, inference code, supporting libraries, metadata) carry the MIT license as shown in the repo’s LICENSE-CODE file [GitHub](https://github.com/deepseek-ai/DeepSeek-V3). + + - **Model parameters** use the custom “LICENSE-MODEL” defined in the repository’s LICENSE-MODEL file [Hugging Face](https://huggingface.co/deepseek-ai/DeepSeek-V3/blob/main/LICENSE-MODEL?utm_source=chatgpt.com). + + - **Evaluation data**, **model card**, and **evaluation results** are marked “unlicensed” following the example pattern when no explicit license is provided. + + - **Technical report** and **research paper** both fall under ArXiv’s perpetual non-exclusive license, as indicated on the arXiv page [arXiv](https://arxiv.org/abs/2412.19437?utm_source=chatgpt.com). + +comments: | + This is reasonably close but doesn't include a Global license. It also mistakes the license file name for the license name. diff --git a/agent/results/test1.yaml:Zone.Identifier b/agent/results/test1.yaml:Zone.Identifier new file mode 100644 index 0000000..e69de29 From ae8005b04a441f6510dcd42a3cd76f5f2e3bb2fc Mon Sep 17 00:00:00 2001 From: Nick Chase Date: Wed, 28 May 2025 11:23:10 -0400 Subject: [PATCH 2/3] Initial Agent commit --- agent/prompt.md:Zone.Identifier | 0 agent/results/test1.yaml:Zone.Identifier | 0 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 agent/prompt.md:Zone.Identifier delete mode 100644 agent/results/test1.yaml:Zone.Identifier diff --git a/agent/prompt.md:Zone.Identifier b/agent/prompt.md:Zone.Identifier deleted file mode 100644 index e69de29..0000000 diff --git a/agent/results/test1.yaml:Zone.Identifier b/agent/results/test1.yaml:Zone.Identifier deleted file mode 100644 index e69de29..0000000 From 7ceb70a5e2c9d7b38692fef8abe1593240a31f71 Mon Sep 17 00:00:00 2001 From: Nick Chase Date: Fri, 30 May 2025 16:37:04 -0400 Subject: [PATCH 3/3] Separate prompts to their own files. --- agent/README.md | 10 +++++++++ agent/instruction-prompt.md | 3 +++ agent/results/test1.yaml | 2 +- agent/{prompt.md => setup-prompt.md} | 31 +--------------------------- 4 files changed, 15 insertions(+), 31 deletions(-) create mode 100644 agent/README.md create mode 100644 agent/instruction-prompt.md rename agent/{prompt.md => setup-prompt.md} (66%) diff --git a/agent/README.md b/agent/README.md new file mode 100644 index 0000000..0f723b7 --- /dev/null +++ b/agent/README.md @@ -0,0 +1,10 @@ +# Introduction and instructions + +We are creating an agent that will update the Model Openness Tool. +We're starting by running the prompt manually to make sure that it works. + +To test, run these prompts in order: + +1. Setup prompt (setup-prompt.md) +2. Instructions prompt (instruction-prompt.md) +3. Model request prompt (the name of the model, such as "DeepSeek-R1") diff --git a/agent/instruction-prompt.md b/agent/instruction-prompt.md new file mode 100644 index 0000000..e32c21f --- /dev/null +++ b/agent/instruction-prompt.md @@ -0,0 +1,3 @@ +Excellent. Now I want you to analyze an open source model. Do any research you have to answer all of these questions about the model and fill out the YAML document. If you don't know the answer, look it up online. DO NOT MAKE ANYTHING UP. Output the full YAML document, then follow up with an explanation of where you found each piece of information. This task is very important to my career. + +Do you understand? diff --git a/agent/results/test1.yaml b/agent/results/test1.yaml index bc0cde9..ae69e11 100644 --- a/agent/results/test1.yaml +++ b/agent/results/test1.yaml @@ -1,6 +1,6 @@ run-number: 1 tester: Nick Chase -prompt-version: 0.1 +prompt-version: 7e2a771 model-used: OpenAI ChatGPT 04-mini model-analyzed: DeepSeek-V3 results-md: | diff --git a/agent/prompt.md b/agent/setup-prompt.md similarity index 66% rename from agent/prompt.md rename to agent/setup-prompt.md index b961cd0..1973423 100644 --- a/agent/prompt.md +++ b/agent/setup-prompt.md @@ -1,19 +1,3 @@ -# Introduction and instructions - -*Current version: 0.1 (Please increment when making changes)* - -We are creating an agent that will update the Model Openness Tool. -We're starting by running the prompt manually to make sure that it works. - -To test, run these prompts in order: - -1. Setup prompt -2. Instructions prompt -3. Model request prompt - -## Setup prompt - -```yaml I'm trying to update information about various open source LLMs for the Model Openness Tool. Read this page, including the tooltips: https://mot-dev.isitopen.ai/model/evaluate This page ultimately populates a YAML document, such as: @@ -76,17 +60,4 @@ release: description: 'The results from evaluating the model' license: unlicensed -Analyze the form and correlate the YAML properties with the form fields. -``` - -## Instructions prompt - -```yaml -Excellent. Now I want you to analyze an open source model. Do any research you have to answer all of these questions about the model and fill out the YAML document. If you don't know the answer, look it up online. DO NOT MAKE ANYTHING UP. Output the full YAML document, then follow up with an explanation of where you found each piece of information. This task is very important to my career. - -Do you understand? -``` - -## Model request prompt - -Here just give the LLM the name of a model, such as `DeepSeek-R1` and it will output the YAML document. +Analyze the form and correlate the YAML properties with the form fields. \ No newline at end of file