From 005db5bdcf9572585a2a902072263136c12de463 Mon Sep 17 00:00:00 2001 From: Kristen Newbury Date: Fri, 12 Dec 2025 17:41:12 -0500 Subject: [PATCH 01/24] Add first version prompt injection query python openai agents sdk --- python/ql/lib/semmle/python/Frameworks.qll | 1 + .../lib/semmle/python/frameworks/OpenAI.qll | 20 ++++++++++ .../PromptInjectionCustomizations.qll | 37 +++++++++++++++++++ .../dataflow/PromptInjectionQuery.qll | 20 ++++++++++ .../Security/CWE-1427/PromptInjection.qhelp | 25 +++++++++++++ .../src/Security/CWE-1427/PromptInjection.ql | 20 ++++++++++ .../PromptInjection.expected | 18 +++++++++ .../PromptInjection.qlref | 1 + .../agent_instructions.py | 12 ++++++ 9 files changed, 154 insertions(+) create mode 100644 python/ql/lib/semmle/python/frameworks/OpenAI.qll create mode 100644 python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll create mode 100644 python/ql/lib/semmle/python/security/dataflow/PromptInjectionQuery.qll create mode 100644 python/ql/src/Security/CWE-1427/PromptInjection.qhelp create mode 100644 python/ql/src/Security/CWE-1427/PromptInjection.ql create mode 100644 python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.expected create mode 100644 python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.qlref create mode 100644 python/ql/test/query-tests/Security/CWE-1427-PromptInjection/agent_instructions.py diff --git a/python/ql/lib/semmle/python/Frameworks.qll b/python/ql/lib/semmle/python/Frameworks.qll index 7694419b41d5..f28686bf2fae 100644 --- a/python/ql/lib/semmle/python/Frameworks.qll +++ b/python/ql/lib/semmle/python/Frameworks.qll @@ -54,6 +54,7 @@ private import semmle.python.frameworks.Multidict private import semmle.python.frameworks.Mysql private import semmle.python.frameworks.MySQLdb private import semmle.python.frameworks.Numpy +private import semmle.python.frameworks.OpenAI private import semmle.python.frameworks.Opml private import semmle.python.frameworks.Oracledb private import semmle.python.frameworks.Pandas diff --git a/python/ql/lib/semmle/python/frameworks/OpenAI.qll b/python/ql/lib/semmle/python/frameworks/OpenAI.qll new file mode 100644 index 000000000000..77a4787a5445 --- /dev/null +++ b/python/ql/lib/semmle/python/frameworks/OpenAI.qll @@ -0,0 +1,20 @@ +/** + * Provides classes modeling security-relevant aspects of the `openAI`Agents SDK package. + * See https://github.com/openai/openai-agents-python. + */ + +private import python +private import semmle.python.ApiGraphs + +/** + * Provides models for Agent (instances of the `agents.Agent` class). + * + * See https://github.com/openai/openai-agents-python. + */ +module Agent { + /** Gets a reference to the `agents.Agent` class. */ + API::Node classRef() { result = API::moduleImport("agents").getMember("Agent") } + + /** Gets a reference to a potential property of `agents.Agent` called instructions which refers to the system prompt. */ + API::Node sink() { result = classRef().getACall().getKeywordParameter("instructions") } +} diff --git a/python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll b/python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll new file mode 100644 index 000000000000..8f1414f5c4a1 --- /dev/null +++ b/python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll @@ -0,0 +1,37 @@ +import python +private import semmle.python.dataflow.new.DataFlow +private import semmle.python.Concepts +private import semmle.python.dataflow.new.RemoteFlowSources +private import semmle.python.dataflow.new.BarrierGuards +private import semmle.python.frameworks.OpenAI + +/** + * Provides default sources, sinks and sanitizers for detecting + * "prompt injection" + * vulnerabilities, as well as extension points for adding your own. + */ +module PromptInjection { + /** + * A data flow source for "prompt injection" vulnerabilities. + */ + abstract class Source extends DataFlow::Node { } + + /** + * A data flow sink for "prompt injection" vulnerabilities. + */ + abstract class Sink extends DataFlow::Node { } + + /** + * A sanitizer for "prompt injection" vulnerabilities. + */ + abstract class Sanitizer extends DataFlow::Node { } + + /** + * An active threat-model source, considered as a flow source. + */ + private class ActiveThreatModelSourceAsSource extends Source, ActiveThreatModelSource { } + + class SystemPromptSink extends Sink { + SystemPromptSink() { this = Agent::sink().asSink() } + } +} diff --git a/python/ql/lib/semmle/python/security/dataflow/PromptInjectionQuery.qll b/python/ql/lib/semmle/python/security/dataflow/PromptInjectionQuery.qll new file mode 100644 index 000000000000..a47d8845c850 --- /dev/null +++ b/python/ql/lib/semmle/python/security/dataflow/PromptInjectionQuery.qll @@ -0,0 +1,20 @@ +private import python +import semmle.python.dataflow.new.DataFlow +import semmle.python.dataflow.new.TaintTracking +import PromptInjectionCustomizations::PromptInjection + +private module PromptInjectionConfig implements DataFlow::ConfigSig { + predicate isSource(DataFlow::Node node) { node instanceof Source } + + predicate isSink(DataFlow::Node node) { + node instanceof Sink + //any() + } + + predicate isBarrierIn(DataFlow::Node node) { node instanceof Sanitizer } + + predicate observeDiffInformedIncrementalMode() { any() } +} + +/** Global taint-tracking for detecting "prompt injection" vulnerabilities. */ +module PromptInjectionFlow = TaintTracking::Global; diff --git a/python/ql/src/Security/CWE-1427/PromptInjection.qhelp b/python/ql/src/Security/CWE-1427/PromptInjection.qhelp new file mode 100644 index 000000000000..b07bae4cca67 --- /dev/null +++ b/python/ql/src/Security/CWE-1427/PromptInjection.qhelp @@ -0,0 +1,25 @@ + + + + +

Prompts can be constructed to bypass the original purposes of an agent and lead to sensitive data leak or +operations that were not intended. +

+ + + +Sanitize user input and also avoid using user input in developer or system level prompts. + + + +

In the following examples, the cases marked GOOD show secure prompt construction; whereas in the case marked BAD they may be susceptible to prompt injection.

+ + + + +
  • OWASP: PromptInjection.
    • +     + +     diff --git a/python/ql/src/Security/CWE-1427/PromptInjection.ql b/python/ql/src/Security/CWE-1427/PromptInjection.ql new file mode 100644 index 000000000000..ab51b681a2f8 --- /dev/null +++ b/python/ql/src/Security/CWE-1427/PromptInjection.ql @@ -0,0 +1,20 @@ +/** + * @name User input used in developer message and or system prompt + * @description User input used in developer message and or system prompt can allow for Prompt Injection attacks. + * @kind path-problem + * @problem.severity error + * @security-severity 5.0 + * @precision high + * @id py/prompt-injection + * @tags security + * external/cwe/cwe-1427 + */ + +import python +import semmle.python.security.dataflow.PromptInjectionQuery +import PromptInjectionFlow::PathGraph + +from PromptInjectionFlow::PathNode source, PromptInjectionFlow::PathNode sink +where PromptInjectionFlow::flowPath(source, sink) +select sink.getNode(), source, sink, "This prompt construction depends on a $@.", source.getNode(), + "user-provided value" diff --git a/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.expected b/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.expected new file mode 100644 index 000000000000..1035cfd357d7 --- /dev/null +++ b/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.expected @@ -0,0 +1,18 @@ +edges +| agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | agent_instructions.py:2:26:2:32 | ControlFlowNode for request | provenance | | +| agent_instructions.py:2:26:2:32 | ControlFlowNode for request | agent_instructions.py:7:13:7:19 | ControlFlowNode for request | provenance | | +| agent_instructions.py:7:5:7:9 | ControlFlowNode for input | agent_instructions.py:9:50:9:89 | ControlFlowNode for BinaryExpr | provenance | | +| agent_instructions.py:7:13:7:19 | ControlFlowNode for request | agent_instructions.py:7:13:7:24 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep | +| agent_instructions.py:7:13:7:24 | ControlFlowNode for Attribute | agent_instructions.py:7:13:7:37 | ControlFlowNode for Attribute() | provenance | dict.get | +| agent_instructions.py:7:13:7:37 | ControlFlowNode for Attribute() | agent_instructions.py:7:5:7:9 | ControlFlowNode for input | provenance | | +nodes +| agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | semmle.label | ControlFlowNode for ImportMember | +| agent_instructions.py:2:26:2:32 | ControlFlowNode for request | semmle.label | ControlFlowNode for request | +| agent_instructions.py:7:5:7:9 | ControlFlowNode for input | semmle.label | ControlFlowNode for input | +| agent_instructions.py:7:13:7:19 | ControlFlowNode for request | semmle.label | ControlFlowNode for request | +| agent_instructions.py:7:13:7:24 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute | +| agent_instructions.py:7:13:7:37 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() | +| agent_instructions.py:9:50:9:89 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr | +subpaths +#select +| agent_instructions.py:9:50:9:89 | ControlFlowNode for BinaryExpr | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | agent_instructions.py:9:50:9:89 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | diff --git a/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.qlref b/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.qlref new file mode 100644 index 000000000000..c72fd0eb3b2c --- /dev/null +++ b/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.qlref @@ -0,0 +1 @@ +query: Security/CWE-1427/PromptInjection.ql \ No newline at end of file diff --git a/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/agent_instructions.py b/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/agent_instructions.py new file mode 100644 index 000000000000..c8e9d6dc23c1 --- /dev/null +++ b/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/agent_instructions.py @@ -0,0 +1,12 @@ +from agents import Agent, Runner +from flask import Flask, request # $ Source=flask +app = Flask(__name__) + +@app.route("/parameter-route") +def get_input(): + input = request.args.get("input") + + agent = Agent(name="Assistant", instructions="This prompt is customized for " + input) # $Alert[py/prompt-injection] + + result = Runner.run_sync(agent, "This is a user message.") + print(result.final_output) \ No newline at end of file From 7a9e03d1be3171f1ed0789cc4dfd10393e25a3f5 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Fri, 2 Jan 2026 12:22:16 +0100 Subject: [PATCH 02/24] Add support for `openai.OpenAI` client library --- python/ql/lib/semmle/python/Frameworks.qll | 1 + .../lib/semmle/python/frameworks/OpenAI.qll | 59 +++++++++++++ .../PromptInjectionCustomizations.qll | 43 ++++++++++ .../dataflow/PromptInjectionQuery.qll | 20 +++++ .../Security/CWE-1427/PromptInjection.qhelp | 25 ++++++ .../src/Security/CWE-1427/PromptInjection.ql | 20 +++++ .../PromptInjection.expected | 80 ++++++++++++++++++ .../PromptInjection.qlref | 1 + .../agent_instructions.py | 12 +++ .../CWE-1427-PromptInjection/openai_test.py | 83 +++++++++++++++++++ 10 files changed, 344 insertions(+) create mode 100644 python/ql/lib/semmle/python/frameworks/OpenAI.qll create mode 100644 python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll create mode 100644 python/ql/lib/semmle/python/security/dataflow/PromptInjectionQuery.qll create mode 100644 python/ql/src/Security/CWE-1427/PromptInjection.qhelp create mode 100644 python/ql/src/Security/CWE-1427/PromptInjection.ql create mode 100644 python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.expected create mode 100644 python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.qlref create mode 100644 python/ql/test/query-tests/Security/CWE-1427-PromptInjection/agent_instructions.py create mode 100644 python/ql/test/query-tests/Security/CWE-1427-PromptInjection/openai_test.py diff --git a/python/ql/lib/semmle/python/Frameworks.qll b/python/ql/lib/semmle/python/Frameworks.qll index 7694419b41d5..f28686bf2fae 100644 --- a/python/ql/lib/semmle/python/Frameworks.qll +++ b/python/ql/lib/semmle/python/Frameworks.qll @@ -54,6 +54,7 @@ private import semmle.python.frameworks.Multidict private import semmle.python.frameworks.Mysql private import semmle.python.frameworks.MySQLdb private import semmle.python.frameworks.Numpy +private import semmle.python.frameworks.OpenAI private import semmle.python.frameworks.Opml private import semmle.python.frameworks.Oracledb private import semmle.python.frameworks.Pandas diff --git a/python/ql/lib/semmle/python/frameworks/OpenAI.qll b/python/ql/lib/semmle/python/frameworks/OpenAI.qll new file mode 100644 index 000000000000..e8abb05ec1ba --- /dev/null +++ b/python/ql/lib/semmle/python/frameworks/OpenAI.qll @@ -0,0 +1,59 @@ +/** + * Provides classes modeling security-relevant aspects of the `openAI`Agents SDK package. + * See https://github.com/openai/openai-agents-python. + */ + +private import python +private import semmle.python.ApiGraphs + +/** + * Provides models for Agent (instances of the `agents.Agent` class). + * + * See https://github.com/openai/openai-agents-python. + */ +module Agent { + /** Gets a reference to the `agents.Agent` class. */ + API::Node classRef() { result = API::moduleImport("agents").getMember("Agent") } + + /** Gets a reference to a potential property of `agents.Agent` called instructions which refers to the system prompt. */ + API::Node sink() { result = classRef().getACall().getKeywordParameter("instructions") } +} + +/** + * Provides models for OpenAI (instances of `openai` classes). + * + * See https://github.com/openai/openai-python. + */ +module OpenAI { + /** Gets a reference to `openai.OpenAI`, `openai.AsyncOpenAI` and `openai.AzureOpenAI`classes. */ + API::Node classRef() { + result = API::moduleImport("openai").getMember(["OpenAI", "AsyncOpenAI", "AzureOpenAI"]) + } + + /** Gets a reference to a potential property of `openai.OpenAI called instructions which refers to the system prompt. */ + API::Node sink() { + result = + classRef() + .getReturn() + .getMember("responses") + .getMember("create") + .getKeywordParameter(["input", "instructions"]) or + result = + classRef() + .getReturn() + .getMember("realtime") + .getMember("connect") + .getReturn() + .getMember("conversation") + .getMember("item") + .getMember("create") + .getKeywordParameter("item") or + result = + classRef() + .getReturn() + .getMember("chat") + .getMember("completions") + .getMember("create") + .getKeywordParameter("messages") + } +} diff --git a/python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll b/python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll new file mode 100644 index 000000000000..a00479c8722c --- /dev/null +++ b/python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll @@ -0,0 +1,43 @@ +import python +private import semmle.python.dataflow.new.DataFlow +private import semmle.python.Concepts +private import semmle.python.dataflow.new.RemoteFlowSources +private import semmle.python.dataflow.new.BarrierGuards +private import semmle.python.frameworks.OpenAI + +/** + * Provides default sources, sinks and sanitizers for detecting + * "prompt injection" + * vulnerabilities, as well as extension points for adding your own. + */ +module PromptInjection { + /** + * A data flow source for "prompt injection" vulnerabilities. + */ + abstract class Source extends DataFlow::Node { } + + /** + * A data flow sink for "prompt injection" vulnerabilities. + */ + abstract class Sink extends DataFlow::Node { } + + /** + * A sanitizer for "prompt injection" vulnerabilities. + */ + abstract class Sanitizer extends DataFlow::Node { } + + /** + * An active threat-model source, considered as a flow source. + */ + private class ActiveThreatModelSourceAsSource extends Source, ActiveThreatModelSource { } + + class SystemPromptSink extends Sink { + SystemPromptSink() { this = Agent::sink().asSink() or this = OpenAI::sink().asSink() } + } + + private import semmle.python.frameworks.data.ModelsAsData + + private class DataAsPromptSink extends Sink { + DataAsPromptSink() { this = ModelOutput::getASinkNode("prompt-injection").asSink() } + } +} diff --git a/python/ql/lib/semmle/python/security/dataflow/PromptInjectionQuery.qll b/python/ql/lib/semmle/python/security/dataflow/PromptInjectionQuery.qll new file mode 100644 index 000000000000..a47d8845c850 --- /dev/null +++ b/python/ql/lib/semmle/python/security/dataflow/PromptInjectionQuery.qll @@ -0,0 +1,20 @@ +private import python +import semmle.python.dataflow.new.DataFlow +import semmle.python.dataflow.new.TaintTracking +import PromptInjectionCustomizations::PromptInjection + +private module PromptInjectionConfig implements DataFlow::ConfigSig { + predicate isSource(DataFlow::Node node) { node instanceof Source } + + predicate isSink(DataFlow::Node node) { + node instanceof Sink + //any() + } + + predicate isBarrierIn(DataFlow::Node node) { node instanceof Sanitizer } + + predicate observeDiffInformedIncrementalMode() { any() } +} + +/** Global taint-tracking for detecting "prompt injection" vulnerabilities. */ +module PromptInjectionFlow = TaintTracking::Global; diff --git a/python/ql/src/Security/CWE-1427/PromptInjection.qhelp b/python/ql/src/Security/CWE-1427/PromptInjection.qhelp new file mode 100644 index 000000000000..b07bae4cca67 --- /dev/null +++ b/python/ql/src/Security/CWE-1427/PromptInjection.qhelp @@ -0,0 +1,25 @@ + + + + +

    Prompts can be constructed to bypass the original purposes of an agent and lead to sensitive data leak or +operations that were not intended. +

    +     + + +Sanitize user input and also avoid using user input in developer or system level prompts. + + + +

    In the following examples, the cases marked GOOD show secure prompt construction; whereas in the case marked BAD they may be susceptible to prompt injection.

    + +     + + +
  • OWASP: PromptInjection.
    • +     + +     diff --git a/python/ql/src/Security/CWE-1427/PromptInjection.ql b/python/ql/src/Security/CWE-1427/PromptInjection.ql new file mode 100644 index 000000000000..ab51b681a2f8 --- /dev/null +++ b/python/ql/src/Security/CWE-1427/PromptInjection.ql @@ -0,0 +1,20 @@ +/** + * @name User input used in developer message and or system prompt + * @description User input used in developer message and or system prompt can allow for Prompt Injection attacks. + * @kind path-problem + * @problem.severity error + * @security-severity 5.0 + * @precision high + * @id py/prompt-injection + * @tags security + * external/cwe/cwe-1427 + */ + +import python +import semmle.python.security.dataflow.PromptInjectionQuery +import PromptInjectionFlow::PathGraph + +from PromptInjectionFlow::PathNode source, PromptInjectionFlow::PathNode sink +where PromptInjectionFlow::flowPath(source, sink) +select sink.getNode(), source, sink, "This prompt construction depends on a $@.", source.getNode(), + "user-provided value" diff --git a/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.expected b/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.expected new file mode 100644 index 000000000000..d163aa2511a0 --- /dev/null +++ b/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.expected @@ -0,0 +1,80 @@ +edges +| agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | agent_instructions.py:2:26:2:32 | ControlFlowNode for request | provenance | | +| agent_instructions.py:2:26:2:32 | ControlFlowNode for request | agent_instructions.py:7:13:7:19 | ControlFlowNode for request | provenance | | +| agent_instructions.py:7:5:7:9 | ControlFlowNode for input | agent_instructions.py:9:50:9:89 | ControlFlowNode for BinaryExpr | provenance | | +| agent_instructions.py:7:13:7:19 | ControlFlowNode for request | agent_instructions.py:7:13:7:24 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep | +| agent_instructions.py:7:13:7:24 | ControlFlowNode for Attribute | agent_instructions.py:7:13:7:37 | ControlFlowNode for Attribute() | provenance | dict.get | +| agent_instructions.py:7:13:7:37 | ControlFlowNode for Attribute() | agent_instructions.py:7:5:7:9 | ControlFlowNode for input | provenance | | +| openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:2:26:2:32 | ControlFlowNode for request | provenance | | +| openai_test.py:2:26:2:32 | ControlFlowNode for request | openai_test.py:12:15:12:21 | ControlFlowNode for request | provenance | | +| openai_test.py:2:26:2:32 | ControlFlowNode for request | openai_test.py:13:13:13:19 | ControlFlowNode for request | provenance | | +| openai_test.py:2:26:2:32 | ControlFlowNode for request | openai_test.py:14:12:14:18 | ControlFlowNode for request | provenance | | +| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:17:22:17:46 | ControlFlowNode for BinaryExpr | provenance | | +| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:22:22:22:46 | ControlFlowNode for BinaryExpr | provenance | | +| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:23:15:36:9 | ControlFlowNode for List | provenance | | +| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:40:22:40:46 | ControlFlowNode for BinaryExpr | provenance | | +| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:58:18:69:9 | ControlFlowNode for List | provenance | | +| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:73:18:82:9 | ControlFlowNode for List | provenance | | +| openai_test.py:12:15:12:21 | ControlFlowNode for request | openai_test.py:12:15:12:26 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep | +| openai_test.py:12:15:12:21 | ControlFlowNode for request | openai_test.py:13:13:13:24 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep | +| openai_test.py:12:15:12:21 | ControlFlowNode for request | openai_test.py:14:12:14:23 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep | +| openai_test.py:12:15:12:26 | ControlFlowNode for Attribute | openai_test.py:12:15:12:41 | ControlFlowNode for Attribute() | provenance | dict.get | +| openai_test.py:12:15:12:41 | ControlFlowNode for Attribute() | openai_test.py:12:5:12:11 | ControlFlowNode for persona | provenance | | +| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:18:15:18:19 | ControlFlowNode for query | provenance | | +| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:23:15:36:9 | ControlFlowNode for List | provenance | | +| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:41:15:41:19 | ControlFlowNode for query | provenance | | +| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:46:18:54:13 | ControlFlowNode for Dict | provenance | | +| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:58:18:69:9 | ControlFlowNode for List | provenance | | +| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:73:18:82:9 | ControlFlowNode for List | provenance | | +| openai_test.py:13:13:13:19 | ControlFlowNode for request | openai_test.py:13:13:13:24 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep | +| openai_test.py:13:13:13:19 | ControlFlowNode for request | openai_test.py:14:12:14:23 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep | +| openai_test.py:13:13:13:24 | ControlFlowNode for Attribute | openai_test.py:13:13:13:37 | ControlFlowNode for Attribute() | provenance | dict.get | +| openai_test.py:13:13:13:37 | ControlFlowNode for Attribute() | openai_test.py:13:5:13:9 | ControlFlowNode for query | provenance | | +| openai_test.py:14:5:14:8 | ControlFlowNode for role | openai_test.py:46:18:54:13 | ControlFlowNode for Dict | provenance | | +| openai_test.py:14:5:14:8 | ControlFlowNode for role | openai_test.py:58:18:69:9 | ControlFlowNode for List | provenance | | +| openai_test.py:14:12:14:18 | ControlFlowNode for request | openai_test.py:14:12:14:23 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep | +| openai_test.py:14:12:14:23 | ControlFlowNode for Attribute | openai_test.py:14:12:14:35 | ControlFlowNode for Attribute() | provenance | dict.get | +| openai_test.py:14:12:14:35 | ControlFlowNode for Attribute() | openai_test.py:14:5:14:8 | ControlFlowNode for role | provenance | | +nodes +| agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | semmle.label | ControlFlowNode for ImportMember | +| agent_instructions.py:2:26:2:32 | ControlFlowNode for request | semmle.label | ControlFlowNode for request | +| agent_instructions.py:7:5:7:9 | ControlFlowNode for input | semmle.label | ControlFlowNode for input | +| agent_instructions.py:7:13:7:19 | ControlFlowNode for request | semmle.label | ControlFlowNode for request | +| agent_instructions.py:7:13:7:24 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute | +| agent_instructions.py:7:13:7:37 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() | +| agent_instructions.py:9:50:9:89 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr | +| openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | semmle.label | ControlFlowNode for ImportMember | +| openai_test.py:2:26:2:32 | ControlFlowNode for request | semmle.label | ControlFlowNode for request | +| openai_test.py:12:5:12:11 | ControlFlowNode for persona | semmle.label | ControlFlowNode for persona | +| openai_test.py:12:15:12:21 | ControlFlowNode for request | semmle.label | ControlFlowNode for request | +| openai_test.py:12:15:12:26 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute | +| openai_test.py:12:15:12:41 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() | +| openai_test.py:13:5:13:9 | ControlFlowNode for query | semmle.label | ControlFlowNode for query | +| openai_test.py:13:13:13:19 | ControlFlowNode for request | semmle.label | ControlFlowNode for request | +| openai_test.py:13:13:13:24 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute | +| openai_test.py:13:13:13:37 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() | +| openai_test.py:14:5:14:8 | ControlFlowNode for role | semmle.label | ControlFlowNode for role | +| openai_test.py:14:12:14:18 | ControlFlowNode for request | semmle.label | ControlFlowNode for request | +| openai_test.py:14:12:14:23 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute | +| openai_test.py:14:12:14:35 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() | +| openai_test.py:17:22:17:46 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr | +| openai_test.py:18:15:18:19 | ControlFlowNode for query | semmle.label | ControlFlowNode for query | +| openai_test.py:22:22:22:46 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr | +| openai_test.py:23:15:36:9 | ControlFlowNode for List | semmle.label | ControlFlowNode for List | +| openai_test.py:40:22:40:46 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr | +| openai_test.py:41:15:41:19 | ControlFlowNode for query | semmle.label | ControlFlowNode for query | +| openai_test.py:46:18:54:13 | ControlFlowNode for Dict | semmle.label | ControlFlowNode for Dict | +| openai_test.py:58:18:69:9 | ControlFlowNode for List | semmle.label | ControlFlowNode for List | +| openai_test.py:73:18:82:9 | ControlFlowNode for List | semmle.label | ControlFlowNode for List | +subpaths +#select +| agent_instructions.py:9:50:9:89 | ControlFlowNode for BinaryExpr | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | agent_instructions.py:9:50:9:89 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:17:22:17:46 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:17:22:17:46 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:18:15:18:19 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:18:15:18:19 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:22:22:22:46 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:22:22:22:46 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:23:15:36:9 | ControlFlowNode for List | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:23:15:36:9 | ControlFlowNode for List | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:40:22:40:46 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:40:22:40:46 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:41:15:41:19 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:41:15:41:19 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:46:18:54:13 | ControlFlowNode for Dict | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:46:18:54:13 | ControlFlowNode for Dict | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:58:18:69:9 | ControlFlowNode for List | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:58:18:69:9 | ControlFlowNode for List | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:73:18:82:9 | ControlFlowNode for List | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:73:18:82:9 | ControlFlowNode for List | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | diff --git a/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.qlref b/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.qlref new file mode 100644 index 000000000000..c72fd0eb3b2c --- /dev/null +++ b/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.qlref @@ -0,0 +1 @@ +query: Security/CWE-1427/PromptInjection.ql \ No newline at end of file diff --git a/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/agent_instructions.py b/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/agent_instructions.py new file mode 100644 index 000000000000..b986fe25a802 --- /dev/null +++ b/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/agent_instructions.py @@ -0,0 +1,12 @@ +from agents import Agent, Runner +from flask import Flask, request # $ Source=flask +app = Flask(__name__) + +@app.route("/parameter-route") +def get_input(): + input = request.args.get("input") + + agent = Agent(name="Assistant", instructions="This prompt is customized for " + input) # $Alert[py/prompt-injection] + + result = Runner.run_sync(agent, "This is a user message.") + print(result.final_output) diff --git a/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/openai_test.py b/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/openai_test.py new file mode 100644 index 000000000000..c95cdeee8c96 --- /dev/null +++ b/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/openai_test.py @@ -0,0 +1,83 @@ +from openai import OpenAI, AsyncOpenAI, AzureOpenAI +from flask import Flask, request # $ Source=flask +app = Flask(__name__) + +client = OpenAI() +async_client = AsyncOpenAI() +azure_client = AzureOpenAI() + + +@app.route("/openai") +def get_input_openai(): + persona = request.args.get("persona") + query = request.args.get("query") + role = request.args.get("role") + + response1 = client.responses.create( + instructions="Talks like a " + persona, # $Alert[py/prompt-injection] + input=query, # $Alert[py/prompt-injection] + ) + + response2 = client.responses.create( + instructions="Talks like a " + persona, # $Alert[py/prompt-injection] + input=[ + { + "role": "developer", + "content": "Talk like a " + persona # $Alert[py/prompt-injection] + }, + { + "role": "user", + "content": [ + {"type": "input_text", + "text": query # $Alert[py/prompt-injection] + } + ] + } + ] + ) + + response3 = await async_client.responses.create( + instructions="Talks like a " + persona, # $Alert[py/prompt-injection] + input=query, # $Alert[py/prompt-injection] + ) + + async with client.realtime.connect(model="gpt-realtime") as connection: + await connection.conversation.item.create( + item={ + "type": "message", + "role": role, # $Alert[py/prompt-injection] + "content": [ + {"type": "input_text", + "text": query # $Alert[py/prompt-injection] + } + ], + } + ) + + completion1 = client.chat.completions.create( + messages=[ + {"role": "developer", + "content": "Talk like a " + persona}, # $Alert[py/prompt-injection] + { + "role": "user", + "content": query, # $Alert[py/prompt-injection] + }, + { + "role": role, # $Alert[py/prompt-injection] + "content": query, # $Alert[py/prompt-injection] + } + ] + ) + + completion2 = azure_client.chat.completions.create( + messages=[ + { + "role": "developer", + "content": "Talk like a " + persona # $Alert[py/prompt-injection] + }, + { + "role": "user", + "content": query, # $Alert[py/prompt-injection] + } + ] + ) From 6c5c87e05075ee49977eadd4f71e0630be300bb9 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Fri, 2 Jan 2026 12:57:03 +0100 Subject: [PATCH 03/24] Fix projcet build errors --- .../change-notes/2026-01-02-prompt-injection.md | 5 +++++ .../dataflow/PromptInjectionCustomizations.qll | 14 +++++++++----- .../security/dataflow/PromptInjectionQuery.qll | 8 ++++++++ .../ql/src/Security/CWE-1427/PromptInjection.qhelp | 5 ++--- 4 files changed, 24 insertions(+), 8 deletions(-) create mode 100644 python/ql/lib/change-notes/2026-01-02-prompt-injection.md diff --git a/python/ql/lib/change-notes/2026-01-02-prompt-injection.md b/python/ql/lib/change-notes/2026-01-02-prompt-injection.md new file mode 100644 index 000000000000..e2bd4674bd57 --- /dev/null +++ b/python/ql/lib/change-notes/2026-01-02-prompt-injection.md @@ -0,0 +1,5 @@ +--- +category: minorAnalysis +--- +* Added propmpt injection query +* Added taint flow model and type model for `agents` and `openai` modules. \ No newline at end of file diff --git a/python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll b/python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll index a00479c8722c..2701ddf6d31c 100644 --- a/python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll +++ b/python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll @@ -1,3 +1,9 @@ +/** + * Provides default sources, sinks and sanitizers for detecting + * "prompt injection" + * vulnerabilities, as well as extension points for adding your own. + */ + import python private import semmle.python.dataflow.new.DataFlow private import semmle.python.Concepts @@ -5,11 +11,6 @@ private import semmle.python.dataflow.new.RemoteFlowSources private import semmle.python.dataflow.new.BarrierGuards private import semmle.python.frameworks.OpenAI -/** - * Provides default sources, sinks and sanitizers for detecting - * "prompt injection" - * vulnerabilities, as well as extension points for adding your own. - */ module PromptInjection { /** * A data flow source for "prompt injection" vulnerabilities. @@ -31,6 +32,9 @@ module PromptInjection { */ private class ActiveThreatModelSourceAsSource extends Source, ActiveThreatModelSource { } + /** + * Agent prompt sinks, considered as a flow sink. + */ class SystemPromptSink extends Sink { SystemPromptSink() { this = Agent::sink().asSink() or this = OpenAI::sink().asSink() } } diff --git a/python/ql/lib/semmle/python/security/dataflow/PromptInjectionQuery.qll b/python/ql/lib/semmle/python/security/dataflow/PromptInjectionQuery.qll index a47d8845c850..6a154f09aaf9 100644 --- a/python/ql/lib/semmle/python/security/dataflow/PromptInjectionQuery.qll +++ b/python/ql/lib/semmle/python/security/dataflow/PromptInjectionQuery.qll @@ -1,3 +1,11 @@ +/** + * Provides taint-tracking configurations for detecting "prompt injection" vulnerabilities. + * + * Note, for performance reasons: only import this file if + * `PromptInjection::Configuration` is needed, otherwise + * `PromptInjectionCustomizations` should be imported instead. + */ + private import python import semmle.python.dataflow.new.DataFlow import semmle.python.dataflow.new.TaintTracking diff --git a/python/ql/src/Security/CWE-1427/PromptInjection.qhelp b/python/ql/src/Security/CWE-1427/PromptInjection.qhelp index b07bae4cca67..aae2ee4bd3e5 100644 --- a/python/ql/src/Security/CWE-1427/PromptInjection.qhelp +++ b/python/ql/src/Security/CWE-1427/PromptInjection.qhelp @@ -5,12 +5,11 @@

    Prompts can be constructed to bypass the original purposes of an agent and lead to sensitive data leak or -operations that were not intended. -

    +operations that were not intended.

     -Sanitize user input and also avoid using user input in developer or system level prompts. +

    Sanitize user input and also avoid using user input in developer or system level prompts.

     From 616698cb4a9c96e53cbc0725693eb37aeac29677 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Fri, 2 Jan 2026 12:59:49 +0100 Subject: [PATCH 04/24] Fix newline at end of PromptInjection.qlref --- .../Security/CWE-1427-PromptInjection/PromptInjection.qlref | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.qlref b/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.qlref index c72fd0eb3b2c..356982d82fc0 100644 --- a/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.qlref +++ b/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.qlref @@ -1 +1 @@ -query: Security/CWE-1427/PromptInjection.ql \ No newline at end of file +query: Security/CWE-1427/PromptInjection.ql From 942834d86f4cbe70fa9208fb34288eae9ce53596 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Fri, 2 Jan 2026 13:05:54 +0100 Subject: [PATCH 05/24] Update python/ql/lib/semmle/python/frameworks/OpenAI.qll Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- python/ql/lib/semmle/python/frameworks/OpenAI.qll | 6 ------ 1 file changed, 6 deletions(-) diff --git a/python/ql/lib/semmle/python/frameworks/OpenAI.qll b/python/ql/lib/semmle/python/frameworks/OpenAI.qll index e8abb05ec1ba..eeb97bbad44f 100644 --- a/python/ql/lib/semmle/python/frameworks/OpenAI.qll +++ b/python/ql/lib/semmle/python/frameworks/OpenAI.qll @@ -25,12 +25,6 @@ module Agent { * See https://github.com/openai/openai-python. */ module OpenAI { - /** Gets a reference to `openai.OpenAI`, `openai.AsyncOpenAI` and `openai.AzureOpenAI`classes. */ - API::Node classRef() { - result = API::moduleImport("openai").getMember(["OpenAI", "AsyncOpenAI", "AzureOpenAI"]) - } - - /** Gets a reference to a potential property of `openai.OpenAI called instructions which refers to the system prompt. */ API::Node sink() { result = classRef() From df979da1b6bd4a54443d91fc537aca5c74397fe6 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Fri, 2 Jan 2026 13:06:05 +0100 Subject: [PATCH 06/24] Update python/ql/src/Security/CWE-1427/PromptInjection.ql Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- python/ql/src/Security/CWE-1427/PromptInjection.ql | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/ql/src/Security/CWE-1427/PromptInjection.ql b/python/ql/src/Security/CWE-1427/PromptInjection.ql index ab51b681a2f8..3a87e96249ae 100644 --- a/python/ql/src/Security/CWE-1427/PromptInjection.ql +++ b/python/ql/src/Security/CWE-1427/PromptInjection.ql @@ -1,6 +1,4 @@ /** - * @name User input used in developer message and or system prompt - * @description User input used in developer message and or system prompt can allow for Prompt Injection attacks. * @kind path-problem * @problem.severity error * @security-severity 5.0 From bacecb7250265341e113378b7306126725faa555 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Fri, 2 Jan 2026 13:32:33 +0100 Subject: [PATCH 07/24] Add example to qlhelp --- .../src/Security/CWE-1427/PromptInjection.qhelp | 3 ++- .../src/Security/CWE-1427/examples/example.py | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 python/ql/src/Security/CWE-1427/examples/example.py diff --git a/python/ql/src/Security/CWE-1427/PromptInjection.qhelp b/python/ql/src/Security/CWE-1427/PromptInjection.qhelp index aae2ee4bd3e5..ec12b58443f1 100644 --- a/python/ql/src/Security/CWE-1427/PromptInjection.qhelp +++ b/python/ql/src/Security/CWE-1427/PromptInjection.qhelp @@ -14,11 +14,12 @@ operations that were not intended.

    In the following examples, the cases marked GOOD show secure prompt construction; whereas in the case marked BAD they may be susceptible to prompt injection.

    - +
  • OWASP: PromptInjection.
    • +
  • OpenAI: Guardrails.
    •  diff --git a/python/ql/src/Security/CWE-1427/examples/example.py b/python/ql/src/Security/CWE-1427/examples/example.py new file mode 100644 index 000000000000..e27da5ad19be --- /dev/null +++ b/python/ql/src/Security/CWE-1427/examples/example.py @@ -0,0 +1,17 @@ +from flask import Flask, request +from agents import Agent, Runner +from guardrails import GuardrailAgent + +@app.route("/parameter-route") +def get_input(): + input = request.args.get("input") + + goodAgent = GuardrailAgent( # GOOD: AGent created with guardrails automatically configured. + config=Path("guardrails_config.json"), + name="Assistant", + instructions="This prompt is customized for " + input) + + badAgent = Agent( + name="Assistant", + instructions="This prompt is customized for " + input # BAD: user input in agent instruction. + ) From a9d0a1639a3cca767650e21d1d016ed2e3ffa48e Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Fri, 2 Jan 2026 13:59:34 +0100 Subject: [PATCH 08/24] Fix missing predicate --- python/ql/lib/semmle/python/frameworks/OpenAI.qll | 6 ++++++ .../dataflow/PromptInjectionCustomizations.qll | 12 ++++++++---- python/ql/src/Security/CWE-1427/PromptInjection.ql | 1 + 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/python/ql/lib/semmle/python/frameworks/OpenAI.qll b/python/ql/lib/semmle/python/frameworks/OpenAI.qll index eeb97bbad44f..e8abb05ec1ba 100644 --- a/python/ql/lib/semmle/python/frameworks/OpenAI.qll +++ b/python/ql/lib/semmle/python/frameworks/OpenAI.qll @@ -25,6 +25,12 @@ module Agent { * See https://github.com/openai/openai-python. */ module OpenAI { + /** Gets a reference to `openai.OpenAI`, `openai.AsyncOpenAI` and `openai.AzureOpenAI`classes. */ + API::Node classRef() { + result = API::moduleImport("openai").getMember(["OpenAI", "AsyncOpenAI", "AzureOpenAI"]) + } + + /** Gets a reference to a potential property of `openai.OpenAI called instructions which refers to the system prompt. */ API::Node sink() { result = classRef() diff --git a/python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll b/python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll index 2701ddf6d31c..a3efa9884c52 100644 --- a/python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll +++ b/python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll @@ -10,7 +10,13 @@ private import semmle.python.Concepts private import semmle.python.dataflow.new.RemoteFlowSources private import semmle.python.dataflow.new.BarrierGuards private import semmle.python.frameworks.OpenAI +private import semmle.python.frameworks.data.ModelsAsData +/** + * Provides default sources, sinks and sanitizers for detecting + * "prompt injection" + * vulnerabilities, as well as extension points for adding your own. + */ module PromptInjection { /** * A data flow source for "prompt injection" vulnerabilities. @@ -39,9 +45,7 @@ module PromptInjection { SystemPromptSink() { this = Agent::sink().asSink() or this = OpenAI::sink().asSink() } } - private import semmle.python.frameworks.data.ModelsAsData - - private class DataAsPromptSink extends Sink { - DataAsPromptSink() { this = ModelOutput::getASinkNode("prompt-injection").asSink() } + private class SinkFromModel extends Sink { + SinkFromModel() { this = ModelOutput::getASinkNode("prompt-injection").asSink() } } } diff --git a/python/ql/src/Security/CWE-1427/PromptInjection.ql b/python/ql/src/Security/CWE-1427/PromptInjection.ql index 3a87e96249ae..323034b0331a 100644 --- a/python/ql/src/Security/CWE-1427/PromptInjection.ql +++ b/python/ql/src/Security/CWE-1427/PromptInjection.ql @@ -1,4 +1,5 @@ /** + * @name Prompt injection * @kind path-problem * @problem.severity error * @security-severity 5.0 From 04193f4bb585eeef90da5c9df35fedd77e3d6558 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Mon, 5 Jan 2026 20:46:42 +0100 Subject: [PATCH 09/24] Une inline expectations --- .../PromptInjectionCustomizations.qll | 7 ++- .../dataflow/PromptInjectionQuery.qll | 9 +-- .../PromptInjection.expected | 56 +++++++++---------- .../PromptInjection.qlref | 1 + .../agent_instructions.py | 2 +- .../CWE-1427-PromptInjection/openai_test.py | 45 +++++++-------- 6 files changed, 62 insertions(+), 58 deletions(-) diff --git a/python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll b/python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll index a3efa9884c52..d22422d53c9c 100644 --- a/python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll +++ b/python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll @@ -42,10 +42,15 @@ module PromptInjection { * Agent prompt sinks, considered as a flow sink. */ class SystemPromptSink extends Sink { - SystemPromptSink() { this = Agent::sink().asSink() or this = OpenAI::sink().asSink() } + SystemPromptSink() { this = [Agent::sink(), OpenAI::sink()].asSink() } } private class SinkFromModel extends Sink { SinkFromModel() { this = ModelOutput::getASinkNode("prompt-injection").asSink() } } + + /** + * A comparison with a constant, considered as a sanitizer-guard. + */ + class ConstCompareAsSanitizerGuard extends Sanitizer, ConstCompareBarrier { } } diff --git a/python/ql/lib/semmle/python/security/dataflow/PromptInjectionQuery.qll b/python/ql/lib/semmle/python/security/dataflow/PromptInjectionQuery.qll index 6a154f09aaf9..5c0413726e62 100644 --- a/python/ql/lib/semmle/python/security/dataflow/PromptInjectionQuery.qll +++ b/python/ql/lib/semmle/python/security/dataflow/PromptInjectionQuery.qll @@ -1,5 +1,5 @@ /** - * Provides taint-tracking configurations for detecting "prompt injection" vulnerabilities. + * Provides a taint-tracking configuration for detecting "prompt injection" vulnerabilities. * * Note, for performance reasons: only import this file if * `PromptInjection::Configuration` is needed, otherwise @@ -14,12 +14,9 @@ import PromptInjectionCustomizations::PromptInjection private module PromptInjectionConfig implements DataFlow::ConfigSig { predicate isSource(DataFlow::Node node) { node instanceof Source } - predicate isSink(DataFlow::Node node) { - node instanceof Sink - //any() - } + predicate isSink(DataFlow::Node node) { node instanceof Sink } - predicate isBarrierIn(DataFlow::Node node) { node instanceof Sanitizer } + predicate isBarrier(DataFlow::Node node) { node instanceof Sanitizer } predicate observeDiffInformedIncrementalMode() { any() } } diff --git a/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.expected b/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.expected index d163aa2511a0..c1b25bb7ca7e 100644 --- a/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.expected +++ b/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.expected @@ -1,3 +1,14 @@ +#select +| agent_instructions.py:9:50:9:89 | ControlFlowNode for BinaryExpr | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | agent_instructions.py:9:50:9:89 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:17:22:17:46 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:17:22:17:46 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:18:15:18:19 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:18:15:18:19 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:22:22:22:46 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:22:22:22:46 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:23:15:37:9 | ControlFlowNode for List | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:23:15:37:9 | ControlFlowNode for List | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:41:22:41:46 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:41:22:41:46 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:42:15:42:19 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:42:15:42:19 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:47:18:55:13 | ControlFlowNode for Dict | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:47:18:55:13 | ControlFlowNode for Dict | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:59:18:70:9 | ControlFlowNode for List | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:59:18:70:9 | ControlFlowNode for List | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:74:18:83:9 | ControlFlowNode for List | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:74:18:83:9 | ControlFlowNode for List | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | edges | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | agent_instructions.py:2:26:2:32 | ControlFlowNode for request | provenance | | | agent_instructions.py:2:26:2:32 | ControlFlowNode for request | agent_instructions.py:7:13:7:19 | ControlFlowNode for request | provenance | | @@ -11,27 +22,27 @@ edges | openai_test.py:2:26:2:32 | ControlFlowNode for request | openai_test.py:14:12:14:18 | ControlFlowNode for request | provenance | | | openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:17:22:17:46 | ControlFlowNode for BinaryExpr | provenance | | | openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:22:22:22:46 | ControlFlowNode for BinaryExpr | provenance | | -| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:23:15:36:9 | ControlFlowNode for List | provenance | | -| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:40:22:40:46 | ControlFlowNode for BinaryExpr | provenance | | -| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:58:18:69:9 | ControlFlowNode for List | provenance | | -| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:73:18:82:9 | ControlFlowNode for List | provenance | | +| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:23:15:37:9 | ControlFlowNode for List | provenance | | +| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:41:22:41:46 | ControlFlowNode for BinaryExpr | provenance | | +| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:59:18:70:9 | ControlFlowNode for List | provenance | | +| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:74:18:83:9 | ControlFlowNode for List | provenance | | | openai_test.py:12:15:12:21 | ControlFlowNode for request | openai_test.py:12:15:12:26 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep | | openai_test.py:12:15:12:21 | ControlFlowNode for request | openai_test.py:13:13:13:24 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep | | openai_test.py:12:15:12:21 | ControlFlowNode for request | openai_test.py:14:12:14:23 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep | | openai_test.py:12:15:12:26 | ControlFlowNode for Attribute | openai_test.py:12:15:12:41 | ControlFlowNode for Attribute() | provenance | dict.get | | openai_test.py:12:15:12:41 | ControlFlowNode for Attribute() | openai_test.py:12:5:12:11 | ControlFlowNode for persona | provenance | | | openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:18:15:18:19 | ControlFlowNode for query | provenance | | -| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:23:15:36:9 | ControlFlowNode for List | provenance | | -| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:41:15:41:19 | ControlFlowNode for query | provenance | | -| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:46:18:54:13 | ControlFlowNode for Dict | provenance | | -| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:58:18:69:9 | ControlFlowNode for List | provenance | | -| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:73:18:82:9 | ControlFlowNode for List | provenance | | +| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:23:15:37:9 | ControlFlowNode for List | provenance | | +| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:42:15:42:19 | ControlFlowNode for query | provenance | | +| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:47:18:55:13 | ControlFlowNode for Dict | provenance | | +| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:59:18:70:9 | ControlFlowNode for List | provenance | | +| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:74:18:83:9 | ControlFlowNode for List | provenance | | | openai_test.py:13:13:13:19 | ControlFlowNode for request | openai_test.py:13:13:13:24 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep | | openai_test.py:13:13:13:19 | ControlFlowNode for request | openai_test.py:14:12:14:23 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep | | openai_test.py:13:13:13:24 | ControlFlowNode for Attribute | openai_test.py:13:13:13:37 | ControlFlowNode for Attribute() | provenance | dict.get | | openai_test.py:13:13:13:37 | ControlFlowNode for Attribute() | openai_test.py:13:5:13:9 | ControlFlowNode for query | provenance | | -| openai_test.py:14:5:14:8 | ControlFlowNode for role | openai_test.py:46:18:54:13 | ControlFlowNode for Dict | provenance | | -| openai_test.py:14:5:14:8 | ControlFlowNode for role | openai_test.py:58:18:69:9 | ControlFlowNode for List | provenance | | +| openai_test.py:14:5:14:8 | ControlFlowNode for role | openai_test.py:47:18:55:13 | ControlFlowNode for Dict | provenance | | +| openai_test.py:14:5:14:8 | ControlFlowNode for role | openai_test.py:59:18:70:9 | ControlFlowNode for List | provenance | | | openai_test.py:14:12:14:18 | ControlFlowNode for request | openai_test.py:14:12:14:23 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep | | openai_test.py:14:12:14:23 | ControlFlowNode for Attribute | openai_test.py:14:12:14:35 | ControlFlowNode for Attribute() | provenance | dict.get | | openai_test.py:14:12:14:35 | ControlFlowNode for Attribute() | openai_test.py:14:5:14:8 | ControlFlowNode for role | provenance | | @@ -60,21 +71,10 @@ nodes | openai_test.py:17:22:17:46 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr | | openai_test.py:18:15:18:19 | ControlFlowNode for query | semmle.label | ControlFlowNode for query | | openai_test.py:22:22:22:46 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr | -| openai_test.py:23:15:36:9 | ControlFlowNode for List | semmle.label | ControlFlowNode for List | -| openai_test.py:40:22:40:46 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr | -| openai_test.py:41:15:41:19 | ControlFlowNode for query | semmle.label | ControlFlowNode for query | -| openai_test.py:46:18:54:13 | ControlFlowNode for Dict | semmle.label | ControlFlowNode for Dict | -| openai_test.py:58:18:69:9 | ControlFlowNode for List | semmle.label | ControlFlowNode for List | -| openai_test.py:73:18:82:9 | ControlFlowNode for List | semmle.label | ControlFlowNode for List | +| openai_test.py:23:15:37:9 | ControlFlowNode for List | semmle.label | ControlFlowNode for List | +| openai_test.py:41:22:41:46 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr | +| openai_test.py:42:15:42:19 | ControlFlowNode for query | semmle.label | ControlFlowNode for query | +| openai_test.py:47:18:55:13 | ControlFlowNode for Dict | semmle.label | ControlFlowNode for Dict | +| openai_test.py:59:18:70:9 | ControlFlowNode for List | semmle.label | ControlFlowNode for List | +| openai_test.py:74:18:83:9 | ControlFlowNode for List | semmle.label | ControlFlowNode for List | subpaths -#select -| agent_instructions.py:9:50:9:89 | ControlFlowNode for BinaryExpr | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | agent_instructions.py:9:50:9:89 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | -| openai_test.py:17:22:17:46 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:17:22:17:46 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | -| openai_test.py:18:15:18:19 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:18:15:18:19 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | -| openai_test.py:22:22:22:46 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:22:22:22:46 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | -| openai_test.py:23:15:36:9 | ControlFlowNode for List | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:23:15:36:9 | ControlFlowNode for List | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | -| openai_test.py:40:22:40:46 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:40:22:40:46 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | -| openai_test.py:41:15:41:19 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:41:15:41:19 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | -| openai_test.py:46:18:54:13 | ControlFlowNode for Dict | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:46:18:54:13 | ControlFlowNode for Dict | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | -| openai_test.py:58:18:69:9 | ControlFlowNode for List | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:58:18:69:9 | ControlFlowNode for List | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | -| openai_test.py:73:18:82:9 | ControlFlowNode for List | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:73:18:82:9 | ControlFlowNode for List | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | diff --git a/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.qlref b/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.qlref index 356982d82fc0..04d00b1b4b8c 100644 --- a/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.qlref +++ b/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.qlref @@ -1 +1,2 @@ query: Security/CWE-1427/PromptInjection.ql +postprocess: utils/test/InlineExpectationsTestQuery.ql \ No newline at end of file diff --git a/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/agent_instructions.py b/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/agent_instructions.py index b986fe25a802..ebe26942ce43 100644 --- a/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/agent_instructions.py +++ b/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/agent_instructions.py @@ -1,5 +1,5 @@ from agents import Agent, Runner -from flask import Flask, request # $ Source=flask +from flask import Flask, request # $ Source app = Flask(__name__) @app.route("/parameter-route") diff --git a/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/openai_test.py b/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/openai_test.py index c95cdeee8c96..f5b29a25dd56 100644 --- a/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/openai_test.py +++ b/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/openai_test.py @@ -1,5 +1,5 @@ from openai import OpenAI, AsyncOpenAI, AzureOpenAI -from flask import Flask, request # $ Source=flask +from flask import Flask, request # $ Source app = Flask(__name__) client = OpenAI() @@ -14,70 +14,71 @@ def get_input_openai(): role = request.args.get("role") response1 = client.responses.create( - instructions="Talks like a " + persona, # $Alert[py/prompt-injection] - input=query, # $Alert[py/prompt-injection] + instructions="Talks like a " + persona, # $ Alert[py/prompt-injection] + input=query, # $ Alert[py/prompt-injection] ) response2 = client.responses.create( - instructions="Talks like a " + persona, # $Alert[py/prompt-injection] + instructions="Talks like a " + persona, # $ Alert[py/prompt-injection] input=[ { "role": "developer", - "content": "Talk like a " + persona # $Alert[py/prompt-injection] + "content": "Talk like a " + persona }, { "role": "user", "content": [ - {"type": "input_text", - "text": query # $Alert[py/prompt-injection] - } + { + "type": "input_text", + "text": query + } ] } - ] + ] # $ Alert[py/prompt-injection] ) response3 = await async_client.responses.create( - instructions="Talks like a " + persona, # $Alert[py/prompt-injection] - input=query, # $Alert[py/prompt-injection] + instructions="Talks like a " + persona, # $ Alert[py/prompt-injection] + input=query, # $ Alert[py/prompt-injection] ) async with client.realtime.connect(model="gpt-realtime") as connection: await connection.conversation.item.create( item={ "type": "message", - "role": role, # $Alert[py/prompt-injection] + "role": role, "content": [ {"type": "input_text", - "text": query # $Alert[py/prompt-injection] + "text": query } ], - } + } # $ Alert[py/prompt-injection] ) completion1 = client.chat.completions.create( messages=[ {"role": "developer", - "content": "Talk like a " + persona}, # $Alert[py/prompt-injection] + "content": "Talk like a " + persona}, { "role": "user", - "content": query, # $Alert[py/prompt-injection] + "content": query, }, { - "role": role, # $Alert[py/prompt-injection] - "content": query, # $Alert[py/prompt-injection] + "role": role, + "content": query, } - ] + ] # $ Alert[py/prompt-injection] ) completion2 = azure_client.chat.completions.create( messages=[ { "role": "developer", - "content": "Talk like a " + persona # $Alert[py/prompt-injection] + "content": "Talk like a " + persona }, { "role": "user", - "content": query, # $Alert[py/prompt-injection] + "content": query, } - ] + ] # $ Alert[py/prompt-injection] ) From 2c83dc3689ed94c8450d74db50b2b278296e12da Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Mon, 5 Jan 2026 22:34:26 +0100 Subject: [PATCH 10/24] Use models as data --- python/ql/lib/semmle/python/Concepts.qll | 25 ++++++++ python/ql/lib/semmle/python/Frameworks.qll | 1 - .../lib/semmle/python/frameworks/OpenAI.qll | 59 ------------------- .../semmle/python/frameworks/agent.model.yml | 6 ++ .../semmle/python/frameworks/openai.model.yml | 14 +++++ .../PromptInjectionCustomizations.qll | 7 +-- .../PromptInjection.expected | 30 +++++----- 7 files changed, 63 insertions(+), 79 deletions(-) delete mode 100644 python/ql/lib/semmle/python/frameworks/OpenAI.qll create mode 100644 python/ql/lib/semmle/python/frameworks/agent.model.yml create mode 100644 python/ql/lib/semmle/python/frameworks/openai.model.yml diff --git a/python/ql/lib/semmle/python/Concepts.qll b/python/ql/lib/semmle/python/Concepts.qll index 0ca8a4dbef01..f7186bc0175b 100644 --- a/python/ql/lib/semmle/python/Concepts.qll +++ b/python/ql/lib/semmle/python/Concepts.qll @@ -325,6 +325,31 @@ private class EncodingAdditionalTaintStep extends TaintTracking::AdditionalTaint } } +/** + * A data-flow node that prompts an AI model. + * + * Extend this class to refine existing API models. If you want to model new APIs, + * extend `AIPrompt::Range` instead. + */ +class AIPrompt extends DataFlow::Node instanceof AIPrompt::Range { + /** Gets an input that is used as AI prompt. */ + DataFlow::Node getAPrompt() { result = super.getAPrompt() } +} + +/** Provides a class for modeling new AI prompting mechanisms. */ +module AIPrompt { + /** + * A data-flow node that prompts an AI model. + * + * Extend this class to model new APIs. If you want to refine existing API models, + * extend `AIPrompt` instead. + */ + abstract class Range extends DataFlow::Node { + /** Gets an input that is logged. */ + abstract DataFlow::Node getAPrompt(); + } +} + /** * A data-flow node that logs data. * diff --git a/python/ql/lib/semmle/python/Frameworks.qll b/python/ql/lib/semmle/python/Frameworks.qll index f28686bf2fae..7694419b41d5 100644 --- a/python/ql/lib/semmle/python/Frameworks.qll +++ b/python/ql/lib/semmle/python/Frameworks.qll @@ -54,7 +54,6 @@ private import semmle.python.frameworks.Multidict private import semmle.python.frameworks.Mysql private import semmle.python.frameworks.MySQLdb private import semmle.python.frameworks.Numpy -private import semmle.python.frameworks.OpenAI private import semmle.python.frameworks.Opml private import semmle.python.frameworks.Oracledb private import semmle.python.frameworks.Pandas diff --git a/python/ql/lib/semmle/python/frameworks/OpenAI.qll b/python/ql/lib/semmle/python/frameworks/OpenAI.qll deleted file mode 100644 index e8abb05ec1ba..000000000000 --- a/python/ql/lib/semmle/python/frameworks/OpenAI.qll +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Provides classes modeling security-relevant aspects of the `openAI`Agents SDK package. - * See https://github.com/openai/openai-agents-python. - */ - -private import python -private import semmle.python.ApiGraphs - -/** - * Provides models for Agent (instances of the `agents.Agent` class). - * - * See https://github.com/openai/openai-agents-python. - */ -module Agent { - /** Gets a reference to the `agents.Agent` class. */ - API::Node classRef() { result = API::moduleImport("agents").getMember("Agent") } - - /** Gets a reference to a potential property of `agents.Agent` called instructions which refers to the system prompt. */ - API::Node sink() { result = classRef().getACall().getKeywordParameter("instructions") } -} - -/** - * Provides models for OpenAI (instances of `openai` classes). - * - * See https://github.com/openai/openai-python. - */ -module OpenAI { - /** Gets a reference to `openai.OpenAI`, `openai.AsyncOpenAI` and `openai.AzureOpenAI`classes. */ - API::Node classRef() { - result = API::moduleImport("openai").getMember(["OpenAI", "AsyncOpenAI", "AzureOpenAI"]) - } - - /** Gets a reference to a potential property of `openai.OpenAI called instructions which refers to the system prompt. */ - API::Node sink() { - result = - classRef() - .getReturn() - .getMember("responses") - .getMember("create") - .getKeywordParameter(["input", "instructions"]) or - result = - classRef() - .getReturn() - .getMember("realtime") - .getMember("connect") - .getReturn() - .getMember("conversation") - .getMember("item") - .getMember("create") - .getKeywordParameter("item") or - result = - classRef() - .getReturn() - .getMember("chat") - .getMember("completions") - .getMember("create") - .getKeywordParameter("messages") - } -} diff --git a/python/ql/lib/semmle/python/frameworks/agent.model.yml b/python/ql/lib/semmle/python/frameworks/agent.model.yml new file mode 100644 index 000000000000..5a923a335197 --- /dev/null +++ b/python/ql/lib/semmle/python/frameworks/agent.model.yml @@ -0,0 +1,6 @@ +extensions: + - addsTo: + pack: codeql/python-all + extensible: sinkModel + data: + - ['agents', 'Member[Agent].Argument[instructions:]', 'prompt-injection'] diff --git a/python/ql/lib/semmle/python/frameworks/openai.model.yml b/python/ql/lib/semmle/python/frameworks/openai.model.yml new file mode 100644 index 000000000000..0f4f15677673 --- /dev/null +++ b/python/ql/lib/semmle/python/frameworks/openai.model.yml @@ -0,0 +1,14 @@ +extensions: + - addsTo: + pack: codeql/python-all + extensible: sinkModel + data: + - ['OpenAI', 'Member[responses].Member[create].Argument[input:,instructions:]', 'prompt-injection'] + - ['OpenAI', 'Member[realtime].Member[connect].ReturnValue.Member[conversation].Member[item].Member[create].Argument[item:]', 'prompt-injection'] + - ['OpenAI', 'Member[chat].Member[completions].Member[create].Argument[messages:]', 'prompt-injection'] + + - addsTo: + pack: codeql/python-all + extensible: typeModel + data: + - ['OpenAI', 'openai', 'Member[OpenAI,AsyncOpenAI,AzureOpenAI].ReturnValue'] diff --git a/python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll b/python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll index d22422d53c9c..5ea14283026b 100644 --- a/python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll +++ b/python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll @@ -9,7 +9,6 @@ private import semmle.python.dataflow.new.DataFlow private import semmle.python.Concepts private import semmle.python.dataflow.new.RemoteFlowSources private import semmle.python.dataflow.new.BarrierGuards -private import semmle.python.frameworks.OpenAI private import semmle.python.frameworks.data.ModelsAsData /** @@ -39,10 +38,10 @@ module PromptInjection { private class ActiveThreatModelSourceAsSource extends Source, ActiveThreatModelSource { } /** - * Agent prompt sinks, considered as a flow sink. + * A prompt to an AI model, considered as a flow sink. */ - class SystemPromptSink extends Sink { - SystemPromptSink() { this = [Agent::sink(), OpenAI::sink()].asSink() } + class AIPromptAsSink extends Sink { + AIPromptAsSink() { this = any(AIPrompt p).getAPrompt() } } private class SinkFromModel extends Sink { diff --git a/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.expected b/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.expected index c1b25bb7ca7e..6742141dfb0f 100644 --- a/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.expected +++ b/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.expected @@ -12,7 +12,7 @@ edges | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | agent_instructions.py:2:26:2:32 | ControlFlowNode for request | provenance | | | agent_instructions.py:2:26:2:32 | ControlFlowNode for request | agent_instructions.py:7:13:7:19 | ControlFlowNode for request | provenance | | -| agent_instructions.py:7:5:7:9 | ControlFlowNode for input | agent_instructions.py:9:50:9:89 | ControlFlowNode for BinaryExpr | provenance | | +| agent_instructions.py:7:5:7:9 | ControlFlowNode for input | agent_instructions.py:9:50:9:89 | ControlFlowNode for BinaryExpr | provenance | Sink:MaD:93 | | agent_instructions.py:7:13:7:19 | ControlFlowNode for request | agent_instructions.py:7:13:7:24 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep | | agent_instructions.py:7:13:7:24 | ControlFlowNode for Attribute | agent_instructions.py:7:13:7:37 | ControlFlowNode for Attribute() | provenance | dict.get | | agent_instructions.py:7:13:7:37 | ControlFlowNode for Attribute() | agent_instructions.py:7:5:7:9 | ControlFlowNode for input | provenance | | @@ -20,29 +20,29 @@ edges | openai_test.py:2:26:2:32 | ControlFlowNode for request | openai_test.py:12:15:12:21 | ControlFlowNode for request | provenance | | | openai_test.py:2:26:2:32 | ControlFlowNode for request | openai_test.py:13:13:13:19 | ControlFlowNode for request | provenance | | | openai_test.py:2:26:2:32 | ControlFlowNode for request | openai_test.py:14:12:14:18 | ControlFlowNode for request | provenance | | -| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:17:22:17:46 | ControlFlowNode for BinaryExpr | provenance | | -| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:22:22:22:46 | ControlFlowNode for BinaryExpr | provenance | | -| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:23:15:37:9 | ControlFlowNode for List | provenance | | -| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:41:22:41:46 | ControlFlowNode for BinaryExpr | provenance | | -| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:59:18:70:9 | ControlFlowNode for List | provenance | | -| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:74:18:83:9 | ControlFlowNode for List | provenance | | +| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:17:22:17:46 | ControlFlowNode for BinaryExpr | provenance | Sink:MaD:58613 | +| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:22:22:22:46 | ControlFlowNode for BinaryExpr | provenance | Sink:MaD:58613 | +| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:23:15:37:9 | ControlFlowNode for List | provenance | Sink:MaD:58613 | +| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:41:22:41:46 | ControlFlowNode for BinaryExpr | provenance | Sink:MaD:58613 | +| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:59:18:70:9 | ControlFlowNode for List | provenance | Sink:MaD:58615 | +| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:74:18:83:9 | ControlFlowNode for List | provenance | Sink:MaD:58615 | | openai_test.py:12:15:12:21 | ControlFlowNode for request | openai_test.py:12:15:12:26 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep | | openai_test.py:12:15:12:21 | ControlFlowNode for request | openai_test.py:13:13:13:24 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep | | openai_test.py:12:15:12:21 | ControlFlowNode for request | openai_test.py:14:12:14:23 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep | | openai_test.py:12:15:12:26 | ControlFlowNode for Attribute | openai_test.py:12:15:12:41 | ControlFlowNode for Attribute() | provenance | dict.get | | openai_test.py:12:15:12:41 | ControlFlowNode for Attribute() | openai_test.py:12:5:12:11 | ControlFlowNode for persona | provenance | | -| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:18:15:18:19 | ControlFlowNode for query | provenance | | -| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:23:15:37:9 | ControlFlowNode for List | provenance | | -| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:42:15:42:19 | ControlFlowNode for query | provenance | | -| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:47:18:55:13 | ControlFlowNode for Dict | provenance | | -| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:59:18:70:9 | ControlFlowNode for List | provenance | | -| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:74:18:83:9 | ControlFlowNode for List | provenance | | +| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:18:15:18:19 | ControlFlowNode for query | provenance | Sink:MaD:58613 | +| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:23:15:37:9 | ControlFlowNode for List | provenance | Sink:MaD:58613 | +| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:42:15:42:19 | ControlFlowNode for query | provenance | Sink:MaD:58613 | +| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:47:18:55:13 | ControlFlowNode for Dict | provenance | Sink:MaD:58614 | +| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:59:18:70:9 | ControlFlowNode for List | provenance | Sink:MaD:58615 | +| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:74:18:83:9 | ControlFlowNode for List | provenance | Sink:MaD:58615 | | openai_test.py:13:13:13:19 | ControlFlowNode for request | openai_test.py:13:13:13:24 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep | | openai_test.py:13:13:13:19 | ControlFlowNode for request | openai_test.py:14:12:14:23 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep | | openai_test.py:13:13:13:24 | ControlFlowNode for Attribute | openai_test.py:13:13:13:37 | ControlFlowNode for Attribute() | provenance | dict.get | | openai_test.py:13:13:13:37 | ControlFlowNode for Attribute() | openai_test.py:13:5:13:9 | ControlFlowNode for query | provenance | | -| openai_test.py:14:5:14:8 | ControlFlowNode for role | openai_test.py:47:18:55:13 | ControlFlowNode for Dict | provenance | | -| openai_test.py:14:5:14:8 | ControlFlowNode for role | openai_test.py:59:18:70:9 | ControlFlowNode for List | provenance | | +| openai_test.py:14:5:14:8 | ControlFlowNode for role | openai_test.py:47:18:55:13 | ControlFlowNode for Dict | provenance | Sink:MaD:58614 | +| openai_test.py:14:5:14:8 | ControlFlowNode for role | openai_test.py:59:18:70:9 | ControlFlowNode for List | provenance | Sink:MaD:58615 | | openai_test.py:14:12:14:18 | ControlFlowNode for request | openai_test.py:14:12:14:23 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep | | openai_test.py:14:12:14:23 | ControlFlowNode for Attribute | openai_test.py:14:12:14:35 | ControlFlowNode for Attribute() | provenance | dict.get | | openai_test.py:14:12:14:35 | ControlFlowNode for Attribute() | openai_test.py:14:5:14:8 | ControlFlowNode for role | provenance | | From 0c7996eb7efe549d0745fc6e50fcf22f1c303b1a Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Tue, 6 Jan 2026 11:39:46 +0100 Subject: [PATCH 11/24] Update python/ql/src/Security/CWE-1427/examples/example.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- python/ql/src/Security/CWE-1427/examples/example.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/ql/src/Security/CWE-1427/examples/example.py b/python/ql/src/Security/CWE-1427/examples/example.py index e27da5ad19be..7319513b53fd 100644 --- a/python/ql/src/Security/CWE-1427/examples/example.py +++ b/python/ql/src/Security/CWE-1427/examples/example.py @@ -6,7 +6,6 @@ def get_input(): input = request.args.get("input") - goodAgent = GuardrailAgent( # GOOD: AGent created with guardrails automatically configured. config=Path("guardrails_config.json"), name="Assistant", instructions="This prompt is customized for " + input) From 21a21469f1afd7decd76e3efee785621223e8530 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Tue, 6 Jan 2026 11:40:51 +0100 Subject: [PATCH 12/24] Update python/ql/lib/semmle/python/Concepts.qll Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- python/ql/lib/semmle/python/Concepts.qll | 1 - 1 file changed, 1 deletion(-) diff --git a/python/ql/lib/semmle/python/Concepts.qll b/python/ql/lib/semmle/python/Concepts.qll index f7186bc0175b..1fc51a06d59b 100644 --- a/python/ql/lib/semmle/python/Concepts.qll +++ b/python/ql/lib/semmle/python/Concepts.qll @@ -345,7 +345,6 @@ module AIPrompt { * extend `AIPrompt` instead. */ abstract class Range extends DataFlow::Node { - /** Gets an input that is logged. */ abstract DataFlow::Node getAPrompt(); } } From 7d450c580b66c3bbd0785549d61dbe3f57985adc Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Tue, 6 Jan 2026 11:41:12 +0100 Subject: [PATCH 13/24] Update python/ql/src/Security/CWE-1427/PromptInjection.qhelp Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- python/ql/src/Security/CWE-1427/PromptInjection.qhelp | 1 - 1 file changed, 1 deletion(-) diff --git a/python/ql/src/Security/CWE-1427/PromptInjection.qhelp b/python/ql/src/Security/CWE-1427/PromptInjection.qhelp index ec12b58443f1..ef6b9c83ac26 100644 --- a/python/ql/src/Security/CWE-1427/PromptInjection.qhelp +++ b/python/ql/src/Security/CWE-1427/PromptInjection.qhelp @@ -18,7 +18,6 @@ operations that were not intended.

     -
  • OWASP: PromptInjection.
  • OpenAI: Guardrails.
    •  From c352ffd28c043484170e146eb4f00f8af0482680 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Tue, 6 Jan 2026 11:41:25 +0100 Subject: [PATCH 14/24] Update python/ql/lib/change-notes/2026-01-02-prompt-injection.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- python/ql/lib/change-notes/2026-01-02-prompt-injection.md | 1 - 1 file changed, 1 deletion(-) diff --git a/python/ql/lib/change-notes/2026-01-02-prompt-injection.md b/python/ql/lib/change-notes/2026-01-02-prompt-injection.md index e2bd4674bd57..22a00c64facc 100644 --- a/python/ql/lib/change-notes/2026-01-02-prompt-injection.md +++ b/python/ql/lib/change-notes/2026-01-02-prompt-injection.md @@ -1,5 +1,4 @@ --- category: minorAnalysis --- -* Added propmpt injection query * Added taint flow model and type model for `agents` and `openai` modules. \ No newline at end of file From 9ea0a1258c306751d047c94ddd90108915333d7a Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Tue, 6 Jan 2026 11:48:14 +0100 Subject: [PATCH 15/24] Fix capitalization typo --- python/ql/src/Security/CWE-1427/examples/example.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/ql/src/Security/CWE-1427/examples/example.py b/python/ql/src/Security/CWE-1427/examples/example.py index 7319513b53fd..a049f727b37a 100644 --- a/python/ql/src/Security/CWE-1427/examples/example.py +++ b/python/ql/src/Security/CWE-1427/examples/example.py @@ -1,11 +1,12 @@ from flask import Flask, request -from agents import Agent, Runner +from agents import Agent from guardrails import GuardrailAgent @app.route("/parameter-route") def get_input(): input = request.args.get("input") + goodAgent = GuardrailAgent( # GOOD: Agent created with guardrails automatically configured. config=Path("guardrails_config.json"), name="Assistant", instructions="This prompt is customized for " + input) From fd8e1700c252249f2470183badc4b92fea3e0780 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Tue, 6 Jan 2026 11:50:41 +0100 Subject: [PATCH 16/24] QLdoc --- python/ql/lib/semmle/python/Concepts.qll | 1 + 1 file changed, 1 insertion(+) diff --git a/python/ql/lib/semmle/python/Concepts.qll b/python/ql/lib/semmle/python/Concepts.qll index 1fc51a06d59b..73d3b0d1e80f 100644 --- a/python/ql/lib/semmle/python/Concepts.qll +++ b/python/ql/lib/semmle/python/Concepts.qll @@ -345,6 +345,7 @@ module AIPrompt { * extend `AIPrompt` instead. */ abstract class Range extends DataFlow::Node { + /** Gets an input that is used as AI prompt. */ abstract DataFlow::Node getAPrompt(); } } From 1a0feb4bac33d6e112d54f6ac8e4c670d083a58a Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Wed, 7 Jan 2026 21:55:58 +0100 Subject: [PATCH 17/24] precise models for experimental query --- .../2026-01-02-prompt-injection.md | 1 + .../semmle/python/frameworks/openai.model.yml | 4 +- .../PromptInjectionCustomizations.qll | 48 ++++++++++++++ .../Security/CWE-1427/PromptInjection.qhelp | 0 .../Security/CWE-1427/PromptInjection.ql | 20 ++++++ .../Security/CWE-1427/examples/example.py | 0 .../PromptInjection.expected | 64 +++++++++---------- .../PromptInjection.qlref | 2 + .../agent_instructions.py | 0 .../CWE-1427-PromptInjection/openai_test.py | 39 ++++++----- 10 files changed, 128 insertions(+), 50 deletions(-) rename python/ql/src/{ => experimental}/Security/CWE-1427/PromptInjection.qhelp (100%) create mode 100644 python/ql/src/experimental/Security/CWE-1427/PromptInjection.ql rename python/ql/src/{ => experimental}/Security/CWE-1427/examples/example.py (100%) rename python/ql/test/{ => experimental}/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.expected (62%) create mode 100644 python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.qlref rename python/ql/test/{ => experimental}/query-tests/Security/CWE-1427-PromptInjection/agent_instructions.py (100%) rename python/ql/test/{ => experimental}/query-tests/Security/CWE-1427-PromptInjection/openai_test.py (65%) diff --git a/python/ql/lib/change-notes/2026-01-02-prompt-injection.md b/python/ql/lib/change-notes/2026-01-02-prompt-injection.md index 22a00c64facc..21f04216ecbc 100644 --- a/python/ql/lib/change-notes/2026-01-02-prompt-injection.md +++ b/python/ql/lib/change-notes/2026-01-02-prompt-injection.md @@ -1,4 +1,5 @@ --- category: minorAnalysis --- +* Added experimental query `py/prompt-injection` to detect potential prompt injection vulnerabilities in code using LLMs. * Added taint flow model and type model for `agents` and `openai` modules. \ No newline at end of file diff --git a/python/ql/lib/semmle/python/frameworks/openai.model.yml b/python/ql/lib/semmle/python/frameworks/openai.model.yml index 0f4f15677673..245d390ab8eb 100644 --- a/python/ql/lib/semmle/python/frameworks/openai.model.yml +++ b/python/ql/lib/semmle/python/frameworks/openai.model.yml @@ -3,9 +3,7 @@ extensions: pack: codeql/python-all extensible: sinkModel data: - - ['OpenAI', 'Member[responses].Member[create].Argument[input:,instructions:]', 'prompt-injection'] - - ['OpenAI', 'Member[realtime].Member[connect].ReturnValue.Member[conversation].Member[item].Member[create].Argument[item:]', 'prompt-injection'] - - ['OpenAI', 'Member[chat].Member[completions].Member[create].Argument[messages:]', 'prompt-injection'] + - ['OpenAI', 'Member[beta].Member[assistants].Member[create].Argument[instructions:]', 'prompt-injection'] - addsTo: pack: codeql/python-all diff --git a/python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll b/python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll index 5ea14283026b..880bf96e12fb 100644 --- a/python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll +++ b/python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll @@ -10,6 +10,7 @@ private import semmle.python.Concepts private import semmle.python.dataflow.new.RemoteFlowSources private import semmle.python.dataflow.new.BarrierGuards private import semmle.python.frameworks.data.ModelsAsData +private import semmle.python.ApiGraphs /** * Provides default sources, sinks and sanitizers for detecting @@ -48,6 +49,53 @@ module PromptInjection { SinkFromModel() { this = ModelOutput::getASinkNode("prompt-injection").asSink() } } + private class PromptContentSink extends Sink { + PromptContentSink() { + exists(API::Node openai, API::Node content | + openai = + API::moduleImport("openai") + .getMember(["OpenAI", "AsyncOpenAI", "AzureOpenAI"]) + .getReturn() and + content = + [ + openai + .getMember("responses") + .getMember("create") + .getKeywordParameter(["input", "instructions"]), + openai + .getMember("responses") + .getMember("create") + .getKeywordParameter(["input", "instructions"]) + .getASubscript() + .getSubscript("content"), + openai + .getMember("realtime") + .getMember("connect") + .getReturn() + .getMember("conversation") + .getMember("item") + .getMember("create") + .getKeywordParameter("item") + .getSubscript("content"), + openai + .getMember("chat") + .getMember("completions") + .getMember("create") + .getKeywordParameter("messages") + .getASubscript() + .getSubscript("content") + ] + | + // content + if not exists(content.getASubscript()) + then this = content.asSink() + else + // content.text + this = content.getASubscript().getSubscript("text").asSink() + ) + } + } + /** * A comparison with a constant, considered as a sanitizer-guard. */ diff --git a/python/ql/src/Security/CWE-1427/PromptInjection.qhelp b/python/ql/src/experimental/Security/CWE-1427/PromptInjection.qhelp similarity index 100% rename from python/ql/src/Security/CWE-1427/PromptInjection.qhelp rename to python/ql/src/experimental/Security/CWE-1427/PromptInjection.qhelp diff --git a/python/ql/src/experimental/Security/CWE-1427/PromptInjection.ql b/python/ql/src/experimental/Security/CWE-1427/PromptInjection.ql new file mode 100644 index 000000000000..3bb985264fac --- /dev/null +++ b/python/ql/src/experimental/Security/CWE-1427/PromptInjection.ql @@ -0,0 +1,20 @@ +/** + * @name Prompt injection + * @kind path-problem + * @problem.severity error + * @security-severity 5.0 + * @precision high + * @id py/prompt-injection + * @tags security + * experimental + * external/cwe/cwe-1427 + */ + +import python +import semmle.python.security.dataflow.PromptInjectionQuery +import PromptInjectionFlow::PathGraph + +from PromptInjectionFlow::PathNode source, PromptInjectionFlow::PathNode sink +where PromptInjectionFlow::flowPath(source, sink) +select sink.getNode(), source, sink, "This prompt construction depends on a $@.", source.getNode(), + "user-provided value" diff --git a/python/ql/src/Security/CWE-1427/examples/example.py b/python/ql/src/experimental/Security/CWE-1427/examples/example.py similarity index 100% rename from python/ql/src/Security/CWE-1427/examples/example.py rename to python/ql/src/experimental/Security/CWE-1427/examples/example.py diff --git a/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.expected b/python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.expected similarity index 62% rename from python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.expected rename to python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.expected index 6742141dfb0f..f52061f3c986 100644 --- a/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.expected +++ b/python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.expected @@ -3,12 +3,17 @@ | openai_test.py:17:22:17:46 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:17:22:17:46 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | | openai_test.py:18:15:18:19 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:18:15:18:19 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | | openai_test.py:22:22:22:46 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:22:22:22:46 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | -| openai_test.py:23:15:37:9 | ControlFlowNode for List | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:23:15:37:9 | ControlFlowNode for List | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:26:28:26:51 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:26:28:26:51 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:33:33:33:37 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:33:33:33:37 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | | openai_test.py:41:22:41:46 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:41:22:41:46 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | | openai_test.py:42:15:42:19 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:42:15:42:19 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | -| openai_test.py:47:18:55:13 | ControlFlowNode for Dict | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:47:18:55:13 | ControlFlowNode for Dict | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | -| openai_test.py:59:18:70:9 | ControlFlowNode for List | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:59:18:70:9 | ControlFlowNode for List | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | -| openai_test.py:74:18:83:9 | ControlFlowNode for List | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:74:18:83:9 | ControlFlowNode for List | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:53:33:53:37 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:53:33:53:37 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:63:28:63:51 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:63:28:63:51 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:67:28:67:32 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:67:28:67:32 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:71:28:71:32 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:71:28:71:32 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:80:28:80:51 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:80:28:80:51 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:84:28:84:32 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:84:28:84:32 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:92:22:92:46 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:92:22:92:46 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | edges | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | agent_instructions.py:2:26:2:32 | ControlFlowNode for request | provenance | | | agent_instructions.py:2:26:2:32 | ControlFlowNode for request | agent_instructions.py:7:13:7:19 | ControlFlowNode for request | provenance | | @@ -19,33 +24,27 @@ edges | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:2:26:2:32 | ControlFlowNode for request | provenance | | | openai_test.py:2:26:2:32 | ControlFlowNode for request | openai_test.py:12:15:12:21 | ControlFlowNode for request | provenance | | | openai_test.py:2:26:2:32 | ControlFlowNode for request | openai_test.py:13:13:13:19 | ControlFlowNode for request | provenance | | -| openai_test.py:2:26:2:32 | ControlFlowNode for request | openai_test.py:14:12:14:18 | ControlFlowNode for request | provenance | | -| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:17:22:17:46 | ControlFlowNode for BinaryExpr | provenance | Sink:MaD:58613 | -| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:22:22:22:46 | ControlFlowNode for BinaryExpr | provenance | Sink:MaD:58613 | -| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:23:15:37:9 | ControlFlowNode for List | provenance | Sink:MaD:58613 | -| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:41:22:41:46 | ControlFlowNode for BinaryExpr | provenance | Sink:MaD:58613 | -| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:59:18:70:9 | ControlFlowNode for List | provenance | Sink:MaD:58615 | -| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:74:18:83:9 | ControlFlowNode for List | provenance | Sink:MaD:58615 | +| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:17:22:17:46 | ControlFlowNode for BinaryExpr | provenance | | +| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:22:22:22:46 | ControlFlowNode for BinaryExpr | provenance | | +| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:26:28:26:51 | ControlFlowNode for BinaryExpr | provenance | | +| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:41:22:41:46 | ControlFlowNode for BinaryExpr | provenance | | +| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:63:28:63:51 | ControlFlowNode for BinaryExpr | provenance | | +| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:80:28:80:51 | ControlFlowNode for BinaryExpr | provenance | | +| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:92:22:92:46 | ControlFlowNode for BinaryExpr | provenance | Sink:MaD:58613 | | openai_test.py:12:15:12:21 | ControlFlowNode for request | openai_test.py:12:15:12:26 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep | | openai_test.py:12:15:12:21 | ControlFlowNode for request | openai_test.py:13:13:13:24 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep | -| openai_test.py:12:15:12:21 | ControlFlowNode for request | openai_test.py:14:12:14:23 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep | | openai_test.py:12:15:12:26 | ControlFlowNode for Attribute | openai_test.py:12:15:12:41 | ControlFlowNode for Attribute() | provenance | dict.get | | openai_test.py:12:15:12:41 | ControlFlowNode for Attribute() | openai_test.py:12:5:12:11 | ControlFlowNode for persona | provenance | | -| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:18:15:18:19 | ControlFlowNode for query | provenance | Sink:MaD:58613 | -| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:23:15:37:9 | ControlFlowNode for List | provenance | Sink:MaD:58613 | -| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:42:15:42:19 | ControlFlowNode for query | provenance | Sink:MaD:58613 | -| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:47:18:55:13 | ControlFlowNode for Dict | provenance | Sink:MaD:58614 | -| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:59:18:70:9 | ControlFlowNode for List | provenance | Sink:MaD:58615 | -| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:74:18:83:9 | ControlFlowNode for List | provenance | Sink:MaD:58615 | +| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:18:15:18:19 | ControlFlowNode for query | provenance | | +| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:33:33:33:37 | ControlFlowNode for query | provenance | | +| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:42:15:42:19 | ControlFlowNode for query | provenance | | +| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:53:33:53:37 | ControlFlowNode for query | provenance | | +| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:67:28:67:32 | ControlFlowNode for query | provenance | | +| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:71:28:71:32 | ControlFlowNode for query | provenance | | +| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:84:28:84:32 | ControlFlowNode for query | provenance | | | openai_test.py:13:13:13:19 | ControlFlowNode for request | openai_test.py:13:13:13:24 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep | -| openai_test.py:13:13:13:19 | ControlFlowNode for request | openai_test.py:14:12:14:23 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep | | openai_test.py:13:13:13:24 | ControlFlowNode for Attribute | openai_test.py:13:13:13:37 | ControlFlowNode for Attribute() | provenance | dict.get | | openai_test.py:13:13:13:37 | ControlFlowNode for Attribute() | openai_test.py:13:5:13:9 | ControlFlowNode for query | provenance | | -| openai_test.py:14:5:14:8 | ControlFlowNode for role | openai_test.py:47:18:55:13 | ControlFlowNode for Dict | provenance | Sink:MaD:58614 | -| openai_test.py:14:5:14:8 | ControlFlowNode for role | openai_test.py:59:18:70:9 | ControlFlowNode for List | provenance | Sink:MaD:58615 | -| openai_test.py:14:12:14:18 | ControlFlowNode for request | openai_test.py:14:12:14:23 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep | -| openai_test.py:14:12:14:23 | ControlFlowNode for Attribute | openai_test.py:14:12:14:35 | ControlFlowNode for Attribute() | provenance | dict.get | -| openai_test.py:14:12:14:35 | ControlFlowNode for Attribute() | openai_test.py:14:5:14:8 | ControlFlowNode for role | provenance | | nodes | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | semmle.label | ControlFlowNode for ImportMember | | agent_instructions.py:2:26:2:32 | ControlFlowNode for request | semmle.label | ControlFlowNode for request | @@ -64,17 +63,18 @@ nodes | openai_test.py:13:13:13:19 | ControlFlowNode for request | semmle.label | ControlFlowNode for request | | openai_test.py:13:13:13:24 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute | | openai_test.py:13:13:13:37 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() | -| openai_test.py:14:5:14:8 | ControlFlowNode for role | semmle.label | ControlFlowNode for role | -| openai_test.py:14:12:14:18 | ControlFlowNode for request | semmle.label | ControlFlowNode for request | -| openai_test.py:14:12:14:23 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute | -| openai_test.py:14:12:14:35 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() | | openai_test.py:17:22:17:46 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr | | openai_test.py:18:15:18:19 | ControlFlowNode for query | semmle.label | ControlFlowNode for query | | openai_test.py:22:22:22:46 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr | -| openai_test.py:23:15:37:9 | ControlFlowNode for List | semmle.label | ControlFlowNode for List | +| openai_test.py:26:28:26:51 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr | +| openai_test.py:33:33:33:37 | ControlFlowNode for query | semmle.label | ControlFlowNode for query | | openai_test.py:41:22:41:46 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr | | openai_test.py:42:15:42:19 | ControlFlowNode for query | semmle.label | ControlFlowNode for query | -| openai_test.py:47:18:55:13 | ControlFlowNode for Dict | semmle.label | ControlFlowNode for Dict | -| openai_test.py:59:18:70:9 | ControlFlowNode for List | semmle.label | ControlFlowNode for List | -| openai_test.py:74:18:83:9 | ControlFlowNode for List | semmle.label | ControlFlowNode for List | +| openai_test.py:53:33:53:37 | ControlFlowNode for query | semmle.label | ControlFlowNode for query | +| openai_test.py:63:28:63:51 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr | +| openai_test.py:67:28:67:32 | ControlFlowNode for query | semmle.label | ControlFlowNode for query | +| openai_test.py:71:28:71:32 | ControlFlowNode for query | semmle.label | ControlFlowNode for query | +| openai_test.py:80:28:80:51 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr | +| openai_test.py:84:28:84:32 | ControlFlowNode for query | semmle.label | ControlFlowNode for query | +| openai_test.py:92:22:92:46 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr | subpaths diff --git a/python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.qlref b/python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.qlref new file mode 100644 index 000000000000..04d00b1b4b8c --- /dev/null +++ b/python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.qlref @@ -0,0 +1,2 @@ +query: Security/CWE-1427/PromptInjection.ql +postprocess: utils/test/InlineExpectationsTestQuery.ql \ No newline at end of file diff --git a/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/agent_instructions.py b/python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/agent_instructions.py similarity index 100% rename from python/ql/test/query-tests/Security/CWE-1427-PromptInjection/agent_instructions.py rename to python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/agent_instructions.py diff --git a/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/openai_test.py b/python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/openai_test.py similarity index 65% rename from python/ql/test/query-tests/Security/CWE-1427-PromptInjection/openai_test.py rename to python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/openai_test.py index f5b29a25dd56..ae4c04633129 100644 --- a/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/openai_test.py +++ b/python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/openai_test.py @@ -23,18 +23,18 @@ def get_input_openai(): input=[ { "role": "developer", - "content": "Talk like a " + persona + "content": "Talk like a " + persona # $ Alert[py/prompt-injection] }, { "role": "user", "content": [ { "type": "input_text", - "text": query + "text": query # $ Alert[py/prompt-injection] } ] } - ] # $ Alert[py/prompt-injection] + ] ) response3 = await async_client.responses.create( @@ -48,37 +48,46 @@ def get_input_openai(): "type": "message", "role": role, "content": [ - {"type": "input_text", - "text": query - } + { + "type": "input_text", + "text": query # $ Alert[py/prompt-injection] + } ], - } # $ Alert[py/prompt-injection] + } ) completion1 = client.chat.completions.create( messages=[ - {"role": "developer", - "content": "Talk like a " + persona}, + { + "role": "developer", + "content": "Talk like a " + persona # $ Alert[py/prompt-injection] + }, { "role": "user", - "content": query, + "content": query, # $ Alert[py/prompt-injection] }, { "role": role, - "content": query, + "content": query, # $ Alert[py/prompt-injection] } - ] # $ Alert[py/prompt-injection] + ] ) completion2 = azure_client.chat.completions.create( messages=[ { "role": "developer", - "content": "Talk like a " + persona + "content": "Talk like a " + persona # $ Alert[py/prompt-injection] }, { "role": "user", - "content": query, + "content": query, # $ Alert[py/prompt-injection] } - ] # $ Alert[py/prompt-injection] + ] + ) + + assistant = client.beta.assistants.create( + name="Test Agent", + model="gpt-4.1", + instructions="Talks like a " + persona # $ Alert[py/prompt-injection] ) From 01b9fa245388341c70a6e4d731f21d94d1fa3d5b Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Wed, 7 Jan 2026 21:59:12 +0100 Subject: [PATCH 18/24] removed spurious file --- .../src/Security/CWE-1427/PromptInjection.ql | 20 ------------------- 1 file changed, 20 deletions(-) delete mode 100644 python/ql/src/Security/CWE-1427/PromptInjection.ql diff --git a/python/ql/src/Security/CWE-1427/PromptInjection.ql b/python/ql/src/Security/CWE-1427/PromptInjection.ql deleted file mode 100644 index 4635b058959f..000000000000 --- a/python/ql/src/Security/CWE-1427/PromptInjection.ql +++ /dev/null @@ -1,20 +0,0 @@ -/** - * @name Prompt injection - * @description User input used in developer message and or system prompt can allow for Prompt Injection attacks. - * @kind path-problem - * @problem.severity error - * @security-severity 5.0 - * @precision high - * @id py/prompt-injection - * @tags security - * external/cwe/cwe-1427 - */ - -import python -import semmle.python.security.dataflow.PromptInjectionQuery -import PromptInjectionFlow::PathGraph - -from PromptInjectionFlow::PathNode source, PromptInjectionFlow::PathNode sink -where PromptInjectionFlow::flowPath(source, sink) -select sink.getNode(), source, sink, "This prompt construction depends on a $@.", source.getNode(), - "user-provided value" From 29aad2e5164accba59bcc5900fdcdeb2ebd20672 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Wed, 7 Jan 2026 22:00:16 +0100 Subject: [PATCH 19/24] remove test --- .../Security/CWE-1427-PromptInjection/PromptInjection.qlref | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.qlref diff --git a/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.qlref b/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.qlref deleted file mode 100644 index c01f22bf45b8..000000000000 --- a/python/ql/test/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.qlref +++ /dev/null @@ -1,2 +0,0 @@ -query: Security/CWE-1427/PromptInjection.ql -postprocess: utils/test/InlineExpectationsTestQuery.ql From 0a36be1ae34ac25ae43819682c6cb4c9e7f1117f Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Wed, 7 Jan 2026 22:44:44 +0100 Subject: [PATCH 20/24] Refactor openai model --- .../lib/semmle/python/frameworks/OpenAI.qll | 57 ++++++++++++++++--- .../PromptInjectionCustomizations.qll | 45 +-------------- 2 files changed, 52 insertions(+), 50 deletions(-) diff --git a/python/ql/lib/semmle/python/frameworks/OpenAI.qll b/python/ql/lib/semmle/python/frameworks/OpenAI.qll index 77a4787a5445..a247645081a4 100644 --- a/python/ql/lib/semmle/python/frameworks/OpenAI.qll +++ b/python/ql/lib/semmle/python/frameworks/OpenAI.qll @@ -7,14 +7,57 @@ private import python private import semmle.python.ApiGraphs /** - * Provides models for Agent (instances of the `agents.Agent` class). + * Provides models for Agent (instances of the `openai.OpenAI` class). * - * See https://github.com/openai/openai-agents-python. + * See https://github.com/openai/openai-python. */ -module Agent { - /** Gets a reference to the `agents.Agent` class. */ - API::Node classRef() { result = API::moduleImport("agents").getMember("Agent") } +module OpenAI { + /** Gets a reference to the `openai.OpenAI` class. */ + API::Node classRef() { + result = + API::moduleImport("openai").getMember(["OpenAI", "AsyncOpenAI", "AzureOpenAI"]).getReturn() + } - /** Gets a reference to a potential property of `agents.Agent` called instructions which refers to the system prompt. */ - API::Node sink() { result = classRef().getACall().getKeywordParameter("instructions") } + /** Gets a reference to a potential property of `openai.OpenAI` called instructions which refers to the system prompt. */ + API::Node getContentNode() { + exists(API::Node content | + content = + classRef() + .getMember("responses") + .getMember("create") + .getKeywordParameter(["input", "instructions"]) or + content = + classRef() + .getMember("responses") + .getMember("create") + .getKeywordParameter(["input", "instructions"]) + .getASubscript() + .getSubscript("content") or + content = + classRef() + .getMember("realtime") + .getMember("connect") + .getReturn() + .getMember("conversation") + .getMember("item") + .getMember("create") + .getKeywordParameter("item") + .getSubscript("content") or + content = + classRef() + .getMember("chat") + .getMember("completions") + .getMember("create") + .getKeywordParameter("messages") + .getASubscript() + .getSubscript("content") + | + // content + if not exists(content.getASubscript()) + then result = content + else + // content.text + result = content.getASubscript().getSubscript("text") + ) + } } diff --git a/python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll b/python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll index 880bf96e12fb..c7f480d47b2e 100644 --- a/python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll +++ b/python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll @@ -10,7 +10,7 @@ private import semmle.python.Concepts private import semmle.python.dataflow.new.RemoteFlowSources private import semmle.python.dataflow.new.BarrierGuards private import semmle.python.frameworks.data.ModelsAsData -private import semmle.python.ApiGraphs +private import semmle.python.frameworks.OpenAI /** * Provides default sources, sinks and sanitizers for detecting @@ -51,48 +51,7 @@ module PromptInjection { private class PromptContentSink extends Sink { PromptContentSink() { - exists(API::Node openai, API::Node content | - openai = - API::moduleImport("openai") - .getMember(["OpenAI", "AsyncOpenAI", "AzureOpenAI"]) - .getReturn() and - content = - [ - openai - .getMember("responses") - .getMember("create") - .getKeywordParameter(["input", "instructions"]), - openai - .getMember("responses") - .getMember("create") - .getKeywordParameter(["input", "instructions"]) - .getASubscript() - .getSubscript("content"), - openai - .getMember("realtime") - .getMember("connect") - .getReturn() - .getMember("conversation") - .getMember("item") - .getMember("create") - .getKeywordParameter("item") - .getSubscript("content"), - openai - .getMember("chat") - .getMember("completions") - .getMember("create") - .getKeywordParameter("messages") - .getASubscript() - .getSubscript("content") - ] - | - // content - if not exists(content.getASubscript()) - then this = content.asSink() - else - // content.text - this = content.getASubscript().getSubscript("text").asSink() - ) + this = OpenAI::getContentNode().asSink() } } From dccaa84b96e557e85a146ff1c9b6a7939a443f66 Mon Sep 17 00:00:00 2001 From: Kristen Newbury Date: Fri, 9 Jan 2026 12:24:20 -0500 Subject: [PATCH 21/24] Improve agents sdk modelling (#5) * Add testcase and coverage for agents sdk runner run with input param * Rename agent sdk module for clarity * Add case for unnamed param use in runner run from agent sdk --- .../lib/semmle/python/frameworks/OpenAI.qll | 23 ++++++++++++++- .../PromptInjectionCustomizations.qll | 2 ++ .../PromptInjection.expected | 14 ++++++++++ .../PromptInjection.qlref | 2 +- .../agent_instructions.py | 28 ++++++++++++++++++- 5 files changed, 66 insertions(+), 3 deletions(-) diff --git a/python/ql/lib/semmle/python/frameworks/OpenAI.qll b/python/ql/lib/semmle/python/frameworks/OpenAI.qll index a247645081a4..ccb8f1bb6b6c 100644 --- a/python/ql/lib/semmle/python/frameworks/OpenAI.qll +++ b/python/ql/lib/semmle/python/frameworks/OpenAI.qll @@ -1,11 +1,32 @@ /** - * Provides classes modeling security-relevant aspects of the `openAI`Agents SDK package. + * Provides classes modeling security-relevant aspects of the `openAI` Agents SDK package. * See https://github.com/openai/openai-agents-python. + * As well as the regular openai python interface. + * See https://github.com/openai/openai-python. */ private import python private import semmle.python.ApiGraphs +/** + * Provides models for agents SDK (instances of the `agents.Runner` class etc). + * + * See https://github.com/openai/openai-agents-python. + */ +module AgentSDK { + /** Gets a reference to the `agents.Agent` class. */ + API::Node classRef() { result = API::moduleImport("agents").getMember("Runner") } + + API::Node runMembers() { result = classRef().getMember(["run", "run_sync", "run_streamed"]) } + + /** Gets a reference to a potential property of `agents.Runner` called input which can refer to a system prompt depending on the role specified. */ + API::Node getContentNode() { + result = runMembers().getKeywordParameter("input").getASubscript().getSubscript("content") + or + result = runMembers().getParameter(_).getASubscript().getSubscript("content") + } +} + /** * Provides models for Agent (instances of the `openai.OpenAI` class). * diff --git a/python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll b/python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll index c7f480d47b2e..aaa8c05418ed 100644 --- a/python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll +++ b/python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll @@ -52,6 +52,8 @@ module PromptInjection { private class PromptContentSink extends Sink { PromptContentSink() { this = OpenAI::getContentNode().asSink() + or + this = AgentSDK::getContentNode().asSink() } } diff --git a/python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.expected b/python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.expected index f52061f3c986..bbf9e5665bab 100644 --- a/python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.expected +++ b/python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.expected @@ -1,5 +1,7 @@ #select | agent_instructions.py:9:50:9:89 | ControlFlowNode for BinaryExpr | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | agent_instructions.py:9:50:9:89 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| agent_instructions.py:25:28:25:32 | ControlFlowNode for input | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | agent_instructions.py:25:28:25:32 | ControlFlowNode for input | This prompt construction depends on a $@. | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| agent_instructions.py:35:28:35:32 | ControlFlowNode for input | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | agent_instructions.py:35:28:35:32 | ControlFlowNode for input | This prompt construction depends on a $@. | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | | openai_test.py:17:22:17:46 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:17:22:17:46 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | | openai_test.py:18:15:18:19 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:18:15:18:19 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | | openai_test.py:22:22:22:46 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:22:22:22:46 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | @@ -17,10 +19,16 @@ edges | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | agent_instructions.py:2:26:2:32 | ControlFlowNode for request | provenance | | | agent_instructions.py:2:26:2:32 | ControlFlowNode for request | agent_instructions.py:7:13:7:19 | ControlFlowNode for request | provenance | | +| agent_instructions.py:2:26:2:32 | ControlFlowNode for request | agent_instructions.py:17:13:17:19 | ControlFlowNode for request | provenance | | | agent_instructions.py:7:5:7:9 | ControlFlowNode for input | agent_instructions.py:9:50:9:89 | ControlFlowNode for BinaryExpr | provenance | Sink:MaD:93 | | agent_instructions.py:7:13:7:19 | ControlFlowNode for request | agent_instructions.py:7:13:7:24 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep | | agent_instructions.py:7:13:7:24 | ControlFlowNode for Attribute | agent_instructions.py:7:13:7:37 | ControlFlowNode for Attribute() | provenance | dict.get | | agent_instructions.py:7:13:7:37 | ControlFlowNode for Attribute() | agent_instructions.py:7:5:7:9 | ControlFlowNode for input | provenance | | +| agent_instructions.py:17:5:17:9 | ControlFlowNode for input | agent_instructions.py:25:28:25:32 | ControlFlowNode for input | provenance | | +| agent_instructions.py:17:5:17:9 | ControlFlowNode for input | agent_instructions.py:35:28:35:32 | ControlFlowNode for input | provenance | | +| agent_instructions.py:17:13:17:19 | ControlFlowNode for request | agent_instructions.py:17:13:17:24 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep | +| agent_instructions.py:17:13:17:24 | ControlFlowNode for Attribute | agent_instructions.py:17:13:17:37 | ControlFlowNode for Attribute() | provenance | dict.get | +| agent_instructions.py:17:13:17:37 | ControlFlowNode for Attribute() | agent_instructions.py:17:5:17:9 | ControlFlowNode for input | provenance | | | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:2:26:2:32 | ControlFlowNode for request | provenance | | | openai_test.py:2:26:2:32 | ControlFlowNode for request | openai_test.py:12:15:12:21 | ControlFlowNode for request | provenance | | | openai_test.py:2:26:2:32 | ControlFlowNode for request | openai_test.py:13:13:13:19 | ControlFlowNode for request | provenance | | @@ -53,6 +61,12 @@ nodes | agent_instructions.py:7:13:7:24 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute | | agent_instructions.py:7:13:7:37 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() | | agent_instructions.py:9:50:9:89 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr | +| agent_instructions.py:17:5:17:9 | ControlFlowNode for input | semmle.label | ControlFlowNode for input | +| agent_instructions.py:17:13:17:19 | ControlFlowNode for request | semmle.label | ControlFlowNode for request | +| agent_instructions.py:17:13:17:24 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute | +| agent_instructions.py:17:13:17:37 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() | +| agent_instructions.py:25:28:25:32 | ControlFlowNode for input | semmle.label | ControlFlowNode for input | +| agent_instructions.py:35:28:35:32 | ControlFlowNode for input | semmle.label | ControlFlowNode for input | | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | semmle.label | ControlFlowNode for ImportMember | | openai_test.py:2:26:2:32 | ControlFlowNode for request | semmle.label | ControlFlowNode for request | | openai_test.py:12:5:12:11 | ControlFlowNode for persona | semmle.label | ControlFlowNode for persona | diff --git a/python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.qlref b/python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.qlref index 04d00b1b4b8c..08466562ffe7 100644 --- a/python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.qlref +++ b/python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.qlref @@ -1,2 +1,2 @@ -query: Security/CWE-1427/PromptInjection.ql +query: experimental/Security/CWE-1427/PromptInjection.ql postprocess: utils/test/InlineExpectationsTestQuery.ql \ No newline at end of file diff --git a/python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/agent_instructions.py b/python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/agent_instructions.py index ebe26942ce43..12cebc1b5831 100644 --- a/python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/agent_instructions.py +++ b/python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/agent_instructions.py @@ -3,10 +3,36 @@ app = Flask(__name__) @app.route("/parameter-route") -def get_input(): +def get_input1(): input = request.args.get("input") agent = Agent(name="Assistant", instructions="This prompt is customized for " + input) # $Alert[py/prompt-injection] result = Runner.run_sync(agent, "This is a user message.") print(result.final_output) + + +@app.route("/parameter-route") +def get_input2(): + input = request.args.get("input") + + agent = Agent(name="Assistant", instructions="This prompt is not customized.") + result = Runner.run_sync( + agent=agent, + input=[ + { + "role": "user", + "content": input, # $Alert[py/prompt-injection] + } + ] + ) + + result2 = Runner.run_sync( + agent, + [ + { + "role": "user", + "content": input, # $Alert[py/prompt-injection] + } + ] + ) From 1ec82d9f56878eb24a27aa0c2d700cfc7e78da70 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Fri, 9 Jan 2026 18:27:41 +0100 Subject: [PATCH 22/24] Update OpenAI.qll --- python/ql/lib/semmle/python/frameworks/OpenAI.qll | 1 + 1 file changed, 1 insertion(+) diff --git a/python/ql/lib/semmle/python/frameworks/OpenAI.qll b/python/ql/lib/semmle/python/frameworks/OpenAI.qll index ccb8f1bb6b6c..601919967cf8 100644 --- a/python/ql/lib/semmle/python/frameworks/OpenAI.qll +++ b/python/ql/lib/semmle/python/frameworks/OpenAI.qll @@ -17,6 +17,7 @@ module AgentSDK { /** Gets a reference to the `agents.Agent` class. */ API::Node classRef() { result = API::moduleImport("agents").getMember("Runner") } + /** Gets a reference to the `run` members. */ API::Node runMembers() { result = classRef().getMember(["run", "run_sync", "run_streamed"]) } /** Gets a reference to a potential property of `agents.Runner` called input which can refer to a system prompt depending on the role specified. */ From 16370d6cd1c38d11114f1ba6de9dd5b073dee7ec Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Fri, 9 Jan 2026 19:07:52 +0100 Subject: [PATCH 23/24] Update python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/openai_test.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../Security/CWE-1427-PromptInjection/openai_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/openai_test.py b/python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/openai_test.py index ae4c04633129..2b25609670c5 100644 --- a/python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/openai_test.py +++ b/python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/openai_test.py @@ -8,7 +8,7 @@ @app.route("/openai") -def get_input_openai(): +async def get_input_openai(): persona = request.args.get("persona") query = request.args.get("query") role = request.args.get("role") From 454268187f87928e1e261648d7e8ea60f59859ea Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Fri, 9 Jan 2026 19:11:41 +0100 Subject: [PATCH 24/24] Update python/ql/lib/semmle/python/frameworks/OpenAI.qll Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- python/ql/lib/semmle/python/frameworks/OpenAI.qll | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ql/lib/semmle/python/frameworks/OpenAI.qll b/python/ql/lib/semmle/python/frameworks/OpenAI.qll index 601919967cf8..7cd11ebabefe 100644 --- a/python/ql/lib/semmle/python/frameworks/OpenAI.qll +++ b/python/ql/lib/semmle/python/frameworks/OpenAI.qll @@ -14,7 +14,7 @@ private import semmle.python.ApiGraphs * See https://github.com/openai/openai-agents-python. */ module AgentSDK { - /** Gets a reference to the `agents.Agent` class. */ + /** Gets a reference to the `agents.Runner` class. */ API::Node classRef() { result = API::moduleImport("agents").getMember("Runner") } /** Gets a reference to the `run` members. */