Skip to content

Python: Temporarily skip OpenAI integration tests #14517

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
SergeyMenshykh merged 3 commits into from
Oct 1, 2026
Merged

Python: Temporarily skip OpenAI integration tests #14517

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8480934
Python: Temporarily skip OpenAI integration tests with invalid creden…
SergeyMenshykh Oct 1, 2026
90849d9
Python: Skip OpenAI agent and multimodality integration cases
SergeyMenshykh Oct 1, 2026
a315799
Python: Skip simple_memory sample while OpenAI key is invalid
SergeyMenshykh Oct 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 45 additions & 10 deletions .../tests/integration/agents/chat_completion_agent/test_chat_completion_agent_integration.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
from semantic_kernel.functions import kernel_function
from tests.integration.agents.agent_test_base import AgentTestBase

openai_api_key_skip = pytest.mark.skip(reason="Temporarily disabled due to invalid OpenAI API key (HTTP 401).")


class WeatherPlugin:
"""A sample Mock weather plugin."""
Expand Down Expand Up @@ -57,15 +59,25 @@ async def chat_completion_agent(self, request):

# region Simple 'Hello' messages tests

@pytest.mark.parametrize("chat_completion_agent", ["azure", "openai"], indirect=True, ids=["azure", "openai"])
@pytest.mark.parametrize(
"chat_completion_agent",
["azure", pytest.param("openai", marks=openai_api_key_skip)],
indirect=True,
ids=["azure", "openai"],
)
async def test_get_response(self, chat_completion_agent: ChatCompletionAgent, agent_test_base: AgentTestBase):
"""Test get response of the agent."""
response = await agent_test_base.get_response_with_retry(chat_completion_agent, messages="Hello")
assert isinstance(response.message, ChatMessageContent)
assert response.message.role == AuthorRole.ASSISTANT
assert response.message.content is not None

@pytest.mark.parametrize("chat_completion_agent", ["azure", "openai"], indirect=True, ids=["azure", "openai"])
@pytest.mark.parametrize(
"chat_completion_agent",
["azure", pytest.param("openai", marks=openai_api_key_skip)],
indirect=True,
ids=["azure", "openai"],
)
async def test_get_response_with_thread(
self, chat_completion_agent: ChatCompletionAgent, agent_test_base: AgentTestBase
):
Expand All @@ -85,7 +97,12 @@ async def test_get_response_with_thread(
assert response.message.content is not None
await thread.delete() if thread else None

@pytest.mark.parametrize("chat_completion_agent", ["azure", "openai"], indirect=True, ids=["azure", "openai"])
@pytest.mark.parametrize(
"chat_completion_agent",
["azure", pytest.param("openai", marks=openai_api_key_skip)],
indirect=True,
ids=["azure", "openai"],
)
async def test_invoke(self, chat_completion_agent: ChatCompletionAgent, agent_test_base: AgentTestBase):
"""Test invoke of the agent."""
responses = await agent_test_base.get_invoke_with_retry(chat_completion_agent, messages="Hello")
Expand All @@ -100,7 +117,12 @@ async def test_invoke(self, chat_completion_agent: ChatCompletionAgent, agent_te
assert usage.prompt_tokens > 0
assert usage.completion_tokens > 0

@pytest.mark.parametrize("chat_completion_agent", ["azure", "openai"], indirect=True, ids=["azure", "openai"])
@pytest.mark.parametrize(
"chat_completion_agent",
["azure", pytest.param("openai", marks=openai_api_key_skip)],
indirect=True,
ids=["azure", "openai"],
)
async def test_invoke_with_thread(self, chat_completion_agent: ChatCompletionAgent, agent_test_base: AgentTestBase):
"""Test invoke of the agent with a thread."""
thread = None
Expand All @@ -119,7 +141,12 @@ async def test_invoke_with_thread(self, chat_completion_agent: ChatCompletionAge

await thread.delete() if thread else None

@pytest.mark.parametrize("chat_completion_agent", ["azure", "openai"], indirect=True, ids=["azure", "openai"])
@pytest.mark.parametrize(
"chat_completion_agent",
["azure", pytest.param("openai", marks=openai_api_key_skip)],
indirect=True,
ids=["azure", "openai"],
)
async def test_invoke_stream(self, chat_completion_agent: ChatCompletionAgent, agent_test_base: AgentTestBase):
"""Test invoke stream of the agent."""
responses = await agent_test_base.get_invoke_stream_with_retry(chat_completion_agent, messages="Hello")
Expand All @@ -134,7 +161,12 @@ async def test_invoke_stream(self, chat_completion_agent: ChatCompletionAgent, a
assert usage.prompt_tokens > 0
assert usage.completion_tokens > 0

@pytest.mark.parametrize("chat_completion_agent", ["azure", "openai"], indirect=True, ids=["azure", "openai"])
@pytest.mark.parametrize(
"chat_completion_agent",
["azure", pytest.param("openai", marks=openai_api_key_skip)],
indirect=True,
ids=["azure", "openai"],
)
async def test_invoke_stream_with_thread(
self, chat_completion_agent: ChatCompletionAgent, agent_test_base: AgentTestBase
):
Expand Down Expand Up @@ -163,7 +195,7 @@ async def test_invoke_stream_with_thread(
"chat_completion_agent",
[
("azure", {"enable_kernel_function": True}),
("openai", {"enable_kernel_function": True}),
pytest.param(("openai", {"enable_kernel_function": True}), marks=openai_api_key_skip),
],
indirect=["chat_completion_agent"],
ids=["azure-function-calling", "openai-function-calling"],
Expand All @@ -185,7 +217,7 @@ async def test_function_calling_get_response(
"chat_completion_agent",
[
("azure", {"enable_kernel_function": True}),
("openai", {"enable_kernel_function": True}),
pytest.param(("openai", {"enable_kernel_function": True}), marks=openai_api_key_skip),
],
indirect=["chat_completion_agent"],
ids=["azure-function-calling", "openai-function-calling"],
Expand All @@ -209,7 +241,7 @@ async def test_function_calling_invoke(
"chat_completion_agent",
[
("azure", {"enable_kernel_function": True}),
("openai", {"enable_kernel_function": True}),
pytest.param(("openai", {"enable_kernel_function": True}), marks=openai_api_key_skip),
],
indirect=["chat_completion_agent"],
ids=["azure-function-calling", "openai-function-calling"],
Expand Down Expand Up @@ -240,7 +272,10 @@ async def test_function_calling_stream(
("azure", {"enable_kernel_function": True}),
pytest.param(
("openai", {"enable_kernel_function": True}),
marks=pytest.mark.xfail(reason="OpenAI service raise error for downloading image from URL"),
marks=[
pytest.mark.xfail(reason="OpenAI service raise error for downloading image from URL"),
openai_api_key_skip,
],
),
],
indirect=["chat_completion_agent"],
Expand Down
64 changes: 48 additions & 16 deletions ...ests/integration/agents/openai_responses_agent/test_openai_responses_agent_integration.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
from semantic_kernel.functions import kernel_function
from tests.integration.agents.agent_test_base import AgentTestBase

openai_api_key_skip = pytest.mark.skip(reason="Temporarily disabled due to invalid OpenAI API key (HTTP 401).")


class WeatherPlugin:
"""A sample Mock weather plugin."""
Expand Down Expand Up @@ -99,7 +101,12 @@ async def responses_agent(self, request):

# region Simple 'Hello' messages tests

@pytest.mark.parametrize("responses_agent", ["azure", "openai"], indirect=True, ids=["azure", "openai"])
@pytest.mark.parametrize(
"responses_agent",
["azure", pytest.param("openai", marks=openai_api_key_skip)],
indirect=True,
ids=["azure", "openai"],
)
async def test_get_response(self, responses_agent: OpenAIResponsesAgent, agent_test_base: AgentTestBase):
"""Test get response of the agent."""
response = await agent_test_base.get_response_with_retry(responses_agent, messages="Hello")
Expand All @@ -108,7 +115,12 @@ async def test_get_response(self, responses_agent: OpenAIResponsesAgent, agent_t
assert response.message.content is not None
assert "thread_id" in response.message.metadata

@pytest.mark.parametrize("responses_agent", ["azure", "openai"], indirect=True, ids=["azure", "openai"])
@pytest.mark.parametrize(
"responses_agent",
["azure", pytest.param("openai", marks=openai_api_key_skip)],
indirect=True,
ids=["azure", "openai"],
)
async def test_get_response_with_thread(
self, responses_agent: OpenAIResponsesAgent, agent_test_base: AgentTestBase
):
Expand All @@ -127,15 +139,25 @@ async def test_get_response_with_thread(

await thread.delete() if thread else None

@pytest.mark.parametrize("responses_agent", ["azure", "openai"], indirect=True, ids=["azure", "openai"])
@pytest.mark.parametrize(
"responses_agent",
["azure", pytest.param("openai", marks=openai_api_key_skip)],
indirect=True,
ids=["azure", "openai"],
)
async def test_invoke(self, responses_agent: OpenAIResponsesAgent, agent_test_base: AgentTestBase):
"""Test invoke of the agent."""
async for response in responses_agent.invoke(messages="Hello"):
assert isinstance(response.message, ChatMessageContent)
assert response.message.role == AuthorRole.ASSISTANT
assert response.message.content is not None

@pytest.mark.parametrize("responses_agent", ["azure", "openai"], indirect=True, ids=["azure", "openai"])
@pytest.mark.parametrize(
"responses_agent",
["azure", pytest.param("openai", marks=openai_api_key_skip)],
indirect=True,
ids=["azure", "openai"],
)
async def test_invoke_with_thread(self, responses_agent: OpenAIResponsesAgent, agent_test_base: AgentTestBase):
"""Test invoke of the agent with a thread."""
thread = None
Expand All @@ -154,7 +176,12 @@ async def test_invoke_with_thread(self, responses_agent: OpenAIResponsesAgent, a

await thread.delete() if thread else None

@pytest.mark.parametrize("responses_agent", ["azure", "openai"], indirect=True, ids=["azure", "openai"])
@pytest.mark.parametrize(
"responses_agent",
["azure", pytest.param("openai", marks=openai_api_key_skip)],
indirect=True,
ids=["azure", "openai"],
)
async def test_invoke_stream(self, responses_agent: OpenAIResponsesAgent, agent_test_base: AgentTestBase):
"""Test invoke stream of the agent."""
responses = await agent_test_base.get_invoke_stream_with_retry(responses_agent, messages="Hello")
Expand All @@ -164,7 +191,12 @@ async def test_invoke_stream(self, responses_agent: OpenAIResponsesAgent, agent_
assert response.message.role == AuthorRole.ASSISTANT
assert response.message.content is not None

@pytest.mark.parametrize("responses_agent", ["azure", "openai"], indirect=True, ids=["azure", "openai"])
@pytest.mark.parametrize(
"responses_agent",
["azure", pytest.param("openai", marks=openai_api_key_skip)],
indirect=True,
ids=["azure", "openai"],
)
async def test_invoke_stream_with_thread(
self, responses_agent: OpenAIResponsesAgent, agent_test_base: AgentTestBase
):
Expand Down Expand Up @@ -193,7 +225,7 @@ async def test_invoke_stream_with_thread(
"responses_agent",
[
# Azure OpenAI Responses API doesn't yet support the web search tool
("openai", {"enable_web_search": True}),
pytest.param(("openai", {"enable_web_search": True}), marks=openai_api_key_skip),
],
indirect=["responses_agent"],
ids=["openai-web-search-get-response"],
Expand All @@ -215,7 +247,7 @@ async def test_web_search_get_response(self, responses_agent: OpenAIResponsesAge
"responses_agent",
[
("azure", {"enable_file_search": True}),
("openai", {"enable_file_search": True}),
pytest.param(("openai", {"enable_file_search": True}), marks=openai_api_key_skip),
],
indirect=["responses_agent"],
ids=["azure-file-search-get-response", "openai-file-search-get-response"],
Expand All @@ -234,7 +266,7 @@ async def test_file_search_get_response(
"responses_agent",
[
("azure", {"enable_file_search": True}),
("openai", {"enable_file_search": True}),
pytest.param(("openai", {"enable_file_search": True}), marks=openai_api_key_skip),
],
indirect=["responses_agent"],
ids=["azure-file-search-invoke", "openai-file-search-invoke"],
Expand All @@ -253,7 +285,7 @@ async def test_file_search_invoke(self, responses_agent: OpenAIResponsesAgent, a
"responses_agent",
[
("azure", {"enable_file_search": True}),
("openai", {"enable_file_search": True}),
pytest.param(("openai", {"enable_file_search": True}), marks=openai_api_key_skip),
],
indirect=["responses_agent"],
ids=["azure-file-search-invoke-stream", "openai-file-search-invoke-stream"],
Expand All @@ -278,7 +310,7 @@ async def test_file_search_invoke_stream(
"responses_agent",
[
("azure", {"enable_kernel_function": True}),
("openai", {"enable_kernel_function": True}),
pytest.param(("openai", {"enable_kernel_function": True}), marks=openai_api_key_skip),
],
indirect=["responses_agent"],
ids=["azure-function-calling-get-response", "openai-function-calling-get-response"],
Expand All @@ -300,7 +332,7 @@ async def test_function_calling_get_response(
"responses_agent",
[
("azure", {"enable_kernel_function": True}),
("openai", {"enable_kernel_function": True}),
pytest.param(("openai", {"enable_kernel_function": True}), marks=openai_api_key_skip),
],
indirect=["responses_agent"],
ids=["azure-function-calling-invoke", "openai-function-calling-invoke"],
Expand All @@ -322,7 +354,7 @@ async def test_function_calling_invoke(self, responses_agent: OpenAIResponsesAge
"responses_agent",
[
("azure", {"enable_kernel_function": True}),
("openai", {"enable_kernel_function": True}),
pytest.param(("openai", {"enable_kernel_function": True}), marks=openai_api_key_skip),
],
indirect=["responses_agent"],
ids=["azure-function-calling-invoke-stream", "openai-function-calling-invoke-stream"],
Expand All @@ -349,7 +381,7 @@ async def test_function_calling_stream(self, responses_agent: OpenAIResponsesAge
"responses_agent",
[
("azure", {"enable_structured_outputs": True}),
("openai", {"enable_structured_outputs": True}),
pytest.param(("openai", {"enable_structured_outputs": True}), marks=openai_api_key_skip),
],
indirect=["responses_agent"],
ids=["azure-structured-outputs-get-response", "openai-structured-outputs-get-response"],
Expand All @@ -371,7 +403,7 @@ async def test_structured_outputs_get_response(
"responses_agent",
[
("azure", {"enable_structured_outputs": True}),
("openai", {"enable_structured_outputs": True}),
pytest.param(("openai", {"enable_structured_outputs": True}), marks=openai_api_key_skip),
],
indirect=["responses_agent"],
ids=["azure-structured-outputs-invoke", "openai-structured-outputs-invoke"],
Expand All @@ -395,7 +427,7 @@ async def test_structured_outputs_invoke(
"responses_agent",
[
("azure", {"enable_structured_outputs": True}),
("openai", {"enable_structured_outputs": True}),
pytest.param(("openai", {"enable_structured_outputs": True}), marks=openai_api_key_skip),
],
indirect=["responses_agent"],
ids=["azure-structured-outputs-invoke-stream", "openai-structured-outputs-invoke-stream"],
Expand Down
1 change: 1 addition & 0 deletions python/tests/integration/audio_to_text/test_audio_to_text.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
"openai",
AudioContent.from_audio_file(os.path.join(os.path.dirname(__file__), "../../", "assets/sample_audio.mp3")),
["hi", "how", "are", "you", "doing"],
marks=pytest.mark.skip(reason="Temporarily disabled due to invalid OpenAI API key (HTTP 401)."),
id="openai",
),
pytest.param(
Expand Down
4 changes: 4 additions & 0 deletions python/tests/integration/completions/test_chat_completion_with_function_calling.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ class FunctionChoiceTestTypes(str, Enum):
]
],
{"test_type": FunctionChoiceTestTypes.AUTO},
marks=pytest.mark.skip(reason="Temporarily disabled due to invalid OpenAI API key (HTTP 401)."),
id="openai_tool_call_auto",
),
pytest.param(
Expand All @@ -97,6 +98,7 @@ class FunctionChoiceTestTypes(str, Enum):
]
],
{"test_type": FunctionChoiceTestTypes.NON_AUTO},
marks=pytest.mark.skip(reason="Temporarily disabled due to invalid OpenAI API key (HTTP 401)."),
id="openai_tool_call_non_auto",
),
pytest.param(
Expand All @@ -123,6 +125,7 @@ class FunctionChoiceTestTypes(str, Enum):
],
],
{"test_type": FunctionChoiceTestTypes.FLOW},
marks=pytest.mark.skip(reason="Temporarily disabled due to invalid OpenAI API key (HTTP 401)."),
id="openai_tool_call_flow",
),
pytest.param(
Expand All @@ -141,6 +144,7 @@ class FunctionChoiceTestTypes(str, Enum):
]
],
{"test_type": FunctionChoiceTestTypes.AUTO},
marks=pytest.mark.skip(reason="Temporarily disabled due to invalid OpenAI API key (HTTP 401)."),
id="openai_tool_call_auto_complex_return_type",
),
# endregion
Expand Down
2 changes: 2 additions & 0 deletions python/tests/integration/completions/test_chat_completion_with_image_input_text_output.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
ChatMessageContent(role=AuthorRole.USER, items=[TextContent(text="Where was it made?")]),
],
{},
marks=pytest.mark.skip(reason="Temporarily disabled due to invalid OpenAI API key (HTTP 401)."),
id="openai_image_input_uri",
),
pytest.param(
Expand All @@ -67,6 +68,7 @@
ChatMessageContent(role=AuthorRole.USER, items=[TextContent(text="Where was it made?")]),
],
{},
marks=pytest.mark.skip(reason="Temporarily disabled due to invalid OpenAI API key (HTTP 401)."),
id="openai_image_input_file",
),
pytest.param(
Expand Down
2 changes: 2 additions & 0 deletions python/tests/integration/completions/test_chat_completions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ class Reasoning(KernelBaseModel):
ChatMessageContent(role=AuthorRole.USER, items=[TextContent(text="How are you today?")]),
],
{},
marks=pytest.mark.skip(reason="Temporarily disabled due to invalid OpenAI API key (HTTP 401)."),
Comment thread
SergeyMenshykh marked this conversation as resolved.
id="openai_text_input",
),
pytest.param(
Expand All @@ -59,6 +60,7 @@ class Reasoning(KernelBaseModel):
ChatMessageContent(role=AuthorRole.USER, items=[TextContent(text="How are you today?")]),
],
{},
marks=pytest.mark.skip(reason="Temporarily disabled due to invalid OpenAI API key (HTTP 401)."),
id="openai_json_schema_response_format",
),
# endregion
Expand Down
Loading
Loading