Christoph SchwörerandClaude Opus 5
28e927013b
Skill 12.0.0 bis 12.2.0 und Adapter 2.0.0 bis 2.2.0. Werkzeugfreigabe: Die Shell-Rechte des OpenCode-Adapters sind jetzt eine Denylist wie beim Claude-Adapter statt einer Allowlist. Ausloeser war der erste Qwen-Lauf, dessen einzige beide Werkzeugaufrufe an einer Pipeline scheiterten. Kontrolltest belegt beide Richtungen: Get-ChildItem | Format-Table laeuft durch, rm wird verweigert, die Datei bleibt bestehen. Lokaler Betrieb: Der Preflight entlaedt alle Modelle vor jedem Lauf und laedt den Kontext auf das Modellmaximum. Gemessen waren zuvor beide Modelle gleichzeitig geladen - 168 MiB frei von 16,3 GB. Qwen 27B passt auf dieser Karte nicht und wurde durch qwen3.5-9b ersetzt, in Q4_K_M wie Gemma. Messinstrument: analyse-anforderungen.py erkennt Feldnamen in Markdown- Fettschrift und Modulpraefixe in IDs. Der erste lokale Lauf mit Artefakten wurde sonst mit null Anforderungen gezaehlt statt mit neun. Regressionsprobe an vier Claude-Laeufen unveraendert. Befunde: Der Standard-Ausgabeblock, mit dem Claude sieben Artefakte erzeugt, liefert bei gemma-4-e4b null von sechs Laeufen ein Ergebnis - das Modell loest den Pfad relativ zum Arbeitsverzeichnis auf. Ein Lauf startete alle sieben vorgesehenen Rollen und lieferte neun formkonforme Anforderungen. Ein anderer erzeugte sieben richtig benannte Dateien ohne eine einzige formkonforme Anforderung. Und eine Shell-Umleitung schrieb an der Denylist vorbei in den eingefrorenen Snapshot - gefunden vom Vorher/Nachher-Vergleich, nicht von der Regel. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
373 lines
14 KiB
Python
373 lines
14 KiB
Python
import importlib.util
|
|
import json
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
|
|
ADAPTER_PATH = Path(__file__).with_name("opencode-adapter.py")
|
|
SPEC = importlib.util.spec_from_file_location("opencode_adapter", ADAPTER_PATH)
|
|
ADAPTER = importlib.util.module_from_spec(SPEC)
|
|
SPEC.loader.exec_module(ADAPTER)
|
|
|
|
|
|
class OpenCodeAdapterTests(unittest.TestCase):
|
|
def test_model_reference_keeps_upstream_slashes(self):
|
|
self.assertEqual(
|
|
(
|
|
"tensorx/qwen/qwen3.8-flash-next",
|
|
"qwen/qwen3.8-flash-next",
|
|
),
|
|
ADAPTER.normalize_model("qwen/qwen3.8-flash-next"),
|
|
)
|
|
self.assertEqual(
|
|
(
|
|
"tensorx/qwen/qwen3.8-flash-next",
|
|
"qwen/qwen3.8-flash-next",
|
|
),
|
|
ADAPTER.normalize_model("tensorx/qwen/qwen3.8-flash-next"),
|
|
)
|
|
|
|
def test_custom_mode_translates_agents_and_restricts_delegation(self):
|
|
base = {
|
|
"provider": {
|
|
"tensorx": {
|
|
"models": {"qwen/qwen3.8-flash-next": {"name": "Qwen"}}
|
|
}
|
|
}
|
|
}
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
temp = Path(temp_dir)
|
|
root = temp / "root"
|
|
output = root / "run" / "Ergebnisse"
|
|
root.mkdir()
|
|
output.mkdir(parents=True)
|
|
agents = temp / "agents.json"
|
|
agents.write_text(
|
|
json.dumps(
|
|
{
|
|
"reviewer": {
|
|
"description": "Prueft Fakten",
|
|
"prompt": "Pruefe nur die zugewiesenen Fakten.",
|
|
}
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
config = ADAPTER.build_run_config(
|
|
base,
|
|
"tensorx/qwen/qwen3.8-flash-next",
|
|
"qwen/qwen3.8-flash-next",
|
|
"custom",
|
|
root,
|
|
output,
|
|
agents,
|
|
)
|
|
|
|
self.assertEqual("allow", config["permission"]["task"]["reviewer"])
|
|
self.assertEqual("deny", config["permission"]["task"]["*"])
|
|
self.assertEqual("subagent", config["agent"]["reviewer"]["mode"])
|
|
self.assertEqual(
|
|
"tensorx/qwen/qwen3.8-flash-next",
|
|
config["agent"]["reviewer"]["model"],
|
|
)
|
|
self.assertEqual("deny", config["agent"]["reviewer"]["permission"]["task"])
|
|
self.assertIn(
|
|
ADAPTER.normalized_path(output) + "/**",
|
|
config["permission"]["edit"],
|
|
)
|
|
self.assertIn(
|
|
"run/Ergebnisse/**",
|
|
config["permission"]["edit"],
|
|
)
|
|
|
|
def test_output_permissions_include_parent_relative_worktree_path(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
temp = Path(temp_dir)
|
|
root = temp / "QuellCode" / "Product"
|
|
output = temp / "Versuche" / "Lauf" / "Ergebnisse"
|
|
root.mkdir(parents=True)
|
|
output.mkdir(parents=True)
|
|
|
|
patterns = ADAPTER.output_permission_patterns(root, output, temp)
|
|
|
|
self.assertIn("Versuche/Lauf/Ergebnisse/**", patterns)
|
|
self.assertIn("../../Versuche/Lauf/Ergebnisse", patterns)
|
|
self.assertIn("../../Versuche/Lauf/Ergebnisse/**", patterns)
|
|
self.assertIn(ADAPTER.normalized_path(output) + "/**", patterns)
|
|
|
|
def test_normalize_result_uses_exported_metrics_and_tool_calls(self):
|
|
session = {
|
|
"info": {
|
|
"id": "ses_test",
|
|
"version": "1.18.25",
|
|
"model": {
|
|
"id": "qwen/qwen3.8-flash-next",
|
|
"providerID": "tensorx",
|
|
},
|
|
"tokens": {
|
|
"input": 100,
|
|
"output": 10,
|
|
"reasoning": 5,
|
|
"cache": {"read": 20, "write": 0},
|
|
},
|
|
"cost": 0.1,
|
|
},
|
|
"messages": [
|
|
{
|
|
"info": {"role": "assistant", "finish": "stop"},
|
|
"parts": [
|
|
{
|
|
"type": "tool",
|
|
"tool": "task",
|
|
"state": {
|
|
"status": "completed",
|
|
"input": {
|
|
"subagent_type": "reviewer",
|
|
"description": "Pruefen",
|
|
},
|
|
},
|
|
},
|
|
{"type": "text", "text": "Fertig"},
|
|
],
|
|
}
|
|
],
|
|
}
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output = Path(temp_dir)
|
|
(output / "StRS.md").write_text("Inhalt", encoding="utf-8")
|
|
result = ADAPTER.normalize_result(
|
|
session=session,
|
|
events=[],
|
|
model_ref="tensorx/qwen/qwen3.8-flash-next",
|
|
mode="custom",
|
|
effort="low",
|
|
exit_code=0,
|
|
timed_out=False,
|
|
interrupted=False,
|
|
duration_s=1.5,
|
|
output_dir=output,
|
|
errors=[],
|
|
)
|
|
|
|
self.assertFalse(result["is_error"])
|
|
self.assertEqual(135, result["usage"]["total_tokens"])
|
|
self.assertEqual(1, result["tool_call_count"])
|
|
self.assertEqual(1, result["subagent_stats"]["spawned"])
|
|
self.assertEqual({"reviewer": 1}, result["subagent_stats"]["by_type"])
|
|
self.assertEqual("Fertig", result["result"])
|
|
self.assertEqual(1, len(result["written_files"]))
|
|
|
|
|
|
class ReadonlyShellTests(unittest.TestCase):
|
|
"""Denylist: alles erlaubt ausser schreibenden und bauenden Kommandos."""
|
|
|
|
def test_catch_all_allow_comes_first(self):
|
|
# OpenCode wertet der Reihe nach aus, die letzte passende Regel gewinnt.
|
|
# Stuende das Catch-all hinten, waere jede Sperre wirkungslos.
|
|
perms = ADAPTER.readonly_shell_permissions()
|
|
self.assertEqual("allow", perms["*"])
|
|
self.assertEqual("*", next(iter(perms)))
|
|
|
|
def test_writing_and_building_commands_are_denied(self):
|
|
perms = ADAPTER.readonly_shell_permissions()
|
|
for muster in ("rm *", "mv *", "sed -i*", "git commit*", "git push*",
|
|
"dotnet *", "msbuild *", "npm install*",
|
|
"Remove-Item *", "Set-Content *", "Out-File *"):
|
|
self.assertEqual("deny", perms[muster], muster)
|
|
|
|
def test_reading_commands_need_no_rule(self):
|
|
# Der Kern der Umstellung: Lesekommandos - auch als Pipeline - sind
|
|
# erlaubt, ohne einzeln aufgefuehrt zu sein. Genau daran scheiterte
|
|
# 'Get-ChildItem ... | Format-Table ...' unter der Allowlist.
|
|
perms = ADAPTER.readonly_shell_permissions()
|
|
for kommando in ("ls -la", "cat foo.cs", "rg muster",
|
|
"Get-ChildItem -Recurse | Format-Table Name",
|
|
"find . -name *.cs"):
|
|
self.assertNotIn(kommando, perms)
|
|
self.assertEqual("allow", perms["*"])
|
|
|
|
def test_no_reading_command_is_denied(self):
|
|
verboten_praefixe = ("ls", "cat", "rg", "grep", "find", "head", "tail",
|
|
"Get-ChildItem", "Get-Content", "Select-String",
|
|
"git status", "git log", "dir", "type")
|
|
for muster, aktion in ADAPTER.readonly_shell_permissions().items():
|
|
if aktion != "deny":
|
|
continue
|
|
for praefix in verboten_praefixe:
|
|
self.assertFalse(
|
|
muster.lower().startswith(praefix.lower()),
|
|
f"Lesekommando gesperrt: {muster}",
|
|
)
|
|
|
|
|
|
class UsageCapturedTests(unittest.TestCase):
|
|
"""Eine Null aus einem laufenden Subagenten ist kein Messwert."""
|
|
|
|
def _result(self, session, output):
|
|
return ADAPTER.normalize_result(
|
|
session=session, events=[], model_ref="lmstudio/google/gemma-4-e4b",
|
|
mode="builtin", effort="high", exit_code=1, timed_out=True,
|
|
interrupted=False, duration_s=3600.0, output_dir=output, errors=[],
|
|
provider="lmstudio", effort_applied=False,
|
|
)
|
|
|
|
def test_running_subagent_with_zero_tokens_is_not_captured(self):
|
|
session = {
|
|
"info": {"model": {"id": "google/gemma-4-e4b"},
|
|
"tokens": {"input": 0, "output": 0, "reasoning": 0,
|
|
"cache": {"read": 0, "write": 0}}},
|
|
"messages": [{
|
|
"info": {"role": "assistant"},
|
|
"parts": [{"type": "tool", "tool": "task",
|
|
"state": {"status": "running",
|
|
"input": {"subagent_type": "explore"}}}],
|
|
}],
|
|
}
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
result = self._result(session, Path(temp_dir))
|
|
self.assertFalse(result["usage_captured"])
|
|
self.assertIn("nicht erfasst", result["usage_note"])
|
|
self.assertIn("1 Subagent(en)", result["usage_note"])
|
|
|
|
def test_real_token_counts_are_marked_captured(self):
|
|
session = {
|
|
"info": {"model": {"id": "google/gemma-4-e4b"},
|
|
"tokens": {"input": 100, "output": 10, "reasoning": 0,
|
|
"cache": {"read": 0, "write": 0}}},
|
|
"messages": [{"info": {"role": "assistant"},
|
|
"parts": [{"type": "text", "text": "ok"}]}],
|
|
}
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
result = self._result(session, Path(temp_dir))
|
|
self.assertTrue(result["usage_captured"])
|
|
self.assertNotIn("usage_note", result)
|
|
|
|
|
|
class OpenCodeLmStudioTests(unittest.TestCase):
|
|
def test_model_reference_uses_lmstudio_provider(self):
|
|
self.assertEqual(
|
|
("lmstudio/google/gemma-4-e4b", "google/gemma-4-e4b"),
|
|
ADAPTER.normalize_model("google/gemma-4-e4b", "lmstudio"),
|
|
)
|
|
self.assertEqual(
|
|
("lmstudio/qwen/qwen3.5-9b", "qwen/qwen3.5-9b"),
|
|
ADAPTER.normalize_model("lmstudio/qwen/qwen3.5-9b", "lmstudio"),
|
|
)
|
|
|
|
def test_template_declares_both_local_models_without_key(self):
|
|
template = json.loads(
|
|
(ADAPTER_PATH.parent / "opencode-lmstudio.json").read_text(
|
|
encoding="utf-8-sig"
|
|
)
|
|
)
|
|
provider = template["provider"]["lmstudio"]
|
|
self.assertEqual(
|
|
"http://localhost:1234/v1", provider["options"]["baseURL"]
|
|
)
|
|
self.assertEqual(
|
|
{"google/gemma-4-e4b", "qwen/qwen3.5-9b"},
|
|
set(provider["models"]),
|
|
)
|
|
# Lokale Server pruefen den Key nicht; er darf nur ein Platzhalter sein.
|
|
self.assertEqual("lm-studio", provider["options"]["apiKey"])
|
|
|
|
def test_build_run_config_pins_loaded_context_window(self):
|
|
base = json.loads(
|
|
(ADAPTER_PATH.parent / "opencode-lmstudio.json").read_text(
|
|
encoding="utf-8-sig"
|
|
)
|
|
)
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = Path(temp_dir) / "root"
|
|
output = root / "run" / "Ergebnisse"
|
|
output.mkdir(parents=True)
|
|
config = ADAPTER.build_run_config(
|
|
base,
|
|
"lmstudio/google/gemma-4-e4b",
|
|
"google/gemma-4-e4b",
|
|
"solo",
|
|
root,
|
|
output,
|
|
None,
|
|
provider="lmstudio",
|
|
context_limit=32768,
|
|
)
|
|
|
|
model = config["provider"]["lmstudio"]["models"]["google/gemma-4-e4b"]
|
|
self.assertEqual(32768, model["limit"]["context"])
|
|
self.assertEqual("deny", config["permission"]["task"])
|
|
self.assertEqual("deny", config["permission"]["webfetch"])
|
|
|
|
def test_normalize_result_reports_local_runtime_and_zero_cost(self):
|
|
runtime = {
|
|
"provider": "lmstudio",
|
|
"quantization": "Q4_K_M",
|
|
"arch": "gemma4",
|
|
"compatibility_type": "gguf",
|
|
"loaded_context_length": 32768,
|
|
"max_context_length": 131072,
|
|
}
|
|
session = {
|
|
"info": {
|
|
"id": "ses_local",
|
|
"model": {"id": "google/gemma-4-e4b"},
|
|
"tokens": {"input": 10, "output": 4, "reasoning": 2,
|
|
"cache": {"read": 0, "write": 0}},
|
|
"cost": 0,
|
|
},
|
|
"messages": [
|
|
{
|
|
"info": {"role": "assistant", "finish": "stop"},
|
|
"parts": [{"type": "text", "text": "Fertig"}],
|
|
}
|
|
],
|
|
}
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output = Path(temp_dir)
|
|
(output / "StRS.md").write_text("Inhalt", encoding="utf-8")
|
|
result = ADAPTER.normalize_result(
|
|
session=session,
|
|
events=[],
|
|
model_ref="lmstudio/google/gemma-4-e4b",
|
|
mode="solo",
|
|
effort="high",
|
|
exit_code=0,
|
|
timed_out=False,
|
|
interrupted=False,
|
|
duration_s=2.0,
|
|
output_dir=output,
|
|
errors=[],
|
|
provider="lmstudio",
|
|
effort_applied=False,
|
|
local_runtime=runtime,
|
|
)
|
|
|
|
self.assertEqual("lmstudio", result["provider"])
|
|
self.assertEqual("opencode-lmstudio", result["adapter"])
|
|
self.assertFalse(result["effort_applied"])
|
|
self.assertEqual(32768, result["context_window"])
|
|
self.assertEqual("Q4_K_M", result["local_runtime"]["quantization"])
|
|
self.assertEqual(0, result["cost"])
|
|
self.assertIn("lokaler Betrieb", result["cost_source"])
|
|
self.assertEqual(16, result["usage"]["total_tokens"])
|
|
|
|
def test_tensorx_result_keeps_remote_shape(self):
|
|
session = {"info": {"model": {"id": "qwen/qwen3.8-flash-next"},
|
|
"tokens": {"input": 1, "output": 1}}, "messages": []}
|
|
result = ADAPTER.normalize_result(
|
|
session=session, events=[], model_ref="tensorx/qwen/qwen3.8-flash-next",
|
|
mode="solo", effort="low", exit_code=0, timed_out=False,
|
|
interrupted=False, duration_s=1.0, output_dir=Path("."), errors=[],
|
|
)
|
|
self.assertEqual("tensorx", result["provider"])
|
|
self.assertEqual("opencode-tensorx", result["adapter"])
|
|
self.assertTrue(result["effort_applied"])
|
|
self.assertNotIn("local_runtime", result)
|
|
self.assertNotIn("context_window", result)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|