Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/main' into experimental-java
Browse files Browse the repository at this point in the history
  • Loading branch information
markwallace-microsoft committed Jul 20, 2023
2 parents ef175ea + f1fb50f commit 8b68683
Show file tree
Hide file tree
Showing 228 changed files with 8,558 additions and 3,847 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/copilot-chat-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ name: Copilot Chat Tests
on:
workflow_dispatch:
push:
branches: ["main", "feature*"]
branches: ["main"]
paths:
- "samples/apps/copilot-chat-app/**"

Expand Down
10 changes: 5 additions & 5 deletions .github/workflows/dotnet-integration-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,24 +23,24 @@ jobs:
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
if: ${{ github.event_name == 'merge_group' }}
if: ${{ github.event_name != 'pull_request' }}
with:
clean: true

- name: Setup .NET
uses: actions/setup-dotnet@v3
if: ${{ github.event_name == 'merge_group' }}
if: ${{ github.event_name != 'pull_request' }}
with:
dotnet-version: 6.0.x

- name: Find projects
shell: bash
if: ${{ github.event_name == 'merge_group' }}
if: ${{ github.event_name != 'pull_request' }}
run: echo "projects=$(find ./dotnet -type f -name "*IntegrationTests.csproj" | tr '\n' ' ')" >> $GITHUB_ENV

- name: Integration Tests
shell: bash
if: ${{ github.event_name == 'merge_group' }}
if: ${{ github.event_name != 'pull_request' }}
env: # Set Azure credentials secret as an input
AzureOpenAI__Label: azure-text-davinci-003
AzureOpenAIEmbedding__Label: azure-text-embedding-ada-002
Expand All @@ -62,4 +62,4 @@ jobs:
with:
name: dotnet-testresults-${{ matrix.configuration }}
path: ./TestResults
if: ${{ github.event_name == 'merge_group' && always() }}
if: ${{ github.event_name != 'pull_request' && always() }}
6 changes: 4 additions & 2 deletions .github/workflows/python-integration-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,13 +38,13 @@ jobs:
export HNSWLIB_NO_NATIVE=1
python -m pip install --upgrade pip setuptools wheel
python -m pip install poetry pytest
cd python && poetry install --with hugging_face --with chromadb --with weaviate
cd python && poetry install
- name: Install dependencies with hnswlib native enabled
if: matrix.os != 'macos-latest' || matrix.python-version != '3.11'
run: |
python -m pip install --upgrade pip setuptools wheel
python -m pip install poetry pytest
cd python && poetry install --with hugging_face --with chromadb --with weaviate
cd python && poetry install
- name: Run Integration Tests
shell: bash
env: # Set Azure credentials secret as an input
Expand All @@ -64,6 +64,8 @@ jobs:
Pinecone__ApiKey: ${{ secrets.PINECONE__APIKEY }}
Pinecone__Environment: ${{ secrets.PINECONE__ENVIRONMENT }}
Postgres__Connectionstr: ${{secrets.POSTGRES__CONNECTIONSTR}}
AZURE_COGNITIVE_SEARCH_ADMIN_KEY: ${{secrets.AZURE_COGNITIVE_SEARCH_ADMIN_KEY}}
AZURE_COGNITIVE_SEARCH_ENDPOINT: ${{secrets.AZURE_COGNITIVE_SEARCH_ENDPOINT}}
run: |
cd python
poetry run pytest ./tests/integration
32 changes: 16 additions & 16 deletions .github/workflows/python-unit-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@ name: Python Unit Tests
on:
workflow_dispatch:
pull_request:
branches: [ "main", "feature*" ]
branches: ["main", "feature*"]
paths:
- 'python/**'
- "python/**"

jobs:
python-unit-tests:
Expand All @@ -14,19 +14,19 @@ jobs:
fail-fast: false
matrix:
python-version: ["3.8", "3.9", "3.10", "3.11"]
os: [ ubuntu-latest, windows-latest, macos-latest ]
os: [ubuntu-latest, windows-latest, macos-latest]

steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
python -m pip install poetry pytest
cd python
poetry install --without chromadb --without hugging_face
- name: Test with pytest
run: |
cd python && poetry run pytest ./tests/unit
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
python -m pip install poetry pytest
cd python
poetry install --without chromadb --without hugging_face --without azure_cognitive_search --without weaviate --without pinecone --without postgres
- name: Test with pytest
run: |
cd python && poetry run pytest ./tests/unit
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -479,3 +479,6 @@ swa-cli.config.json
**/copilot-chat-app/webapp/build
**/copilot-chat-app/webapp/node_modules
**/copilot-chat-app/webapi/data/eng.traineddata

# Semantic Kernel Tools
/.semantic-kernel
8 changes: 8 additions & 0 deletions .vscode/launch.json
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,14 @@
"name": ".NET Core Attach",
"type": "coreclr",
"request": "attach"
},
{
"cwd":"${workspaceFolder}/python",
"name": "Python: Test Module",
"type": "python",
"request": "launch",
"module": "pytest",
"args": ["${file}"],
}
]
}
92 changes: 2 additions & 90 deletions FEATURE_MATRIX.md
Original file line number Diff line number Diff line change
@@ -1,93 +1,5 @@
# Semantic Kernel Feature Matrix by Language

**Legend**
This document can be found on the Semantic Kernel Documentation site on [Supported Languages.](https://learn.microsoft.com/en-us/semantic-kernel/get-started/supported-languages)

✅ - Feature implemented
🔄 - Feature partially implemented (see associated Note column)
❌ - Feature not implemented

## AI Services

| | C# | Python | Java | Notes |
|-----------------------------------|:----:|:------:|:----:|-------|
| Text Generation |||| Example: text-davinci-003 |
| Text Embeddings |||| Example: text-embeddings-ada-002 |
| Chat Completion |||| Example: GPT-4, GPT-3.5-turbo |
| Image Generation |||| Example: Dall-E 2 |

## AI Service Endpoints

| | C# | Python | Java | Notes |
|-----------------------------------|:---:|:------:|:----:|-------|
| OpenAI |||| |
| Azure OpenAI |||| |
| Hugging Face Inference API | 🔄 ||| Coming soon to Python, not all scenarios are covered for .NET |
| Hugging Face Local |||| |
| Custom || 🔄 || Requires to define the service schema in the application |

## Tokenizers

| | C# | Python | Java | Notes |
|-----------------------------------|:---:|:------:|:----:|-------|
| GPT2 |||| |
| GPT3 |||| |
| tiktoken |||| Coming soon. Can be added manually to Python via `pip install tiktoken` |

## Core Skills

| | C# | Python | Java | Notes |
|-----------------------------------|:---:|:------:|:----:|-------|
| TextMemory Skill ||| 🔄 | |
| ConversationSummary Skill |||| |
| FileIO Skill |||| |
| Http Skill |||| |
| Math Skill |||| |
| Text Skill ||| 🔄 | |
| Time Skill |||| |
| Wait Skill |||| |

## Planning

| | C# | Python | Java | Notes |
|-----------------------------------|:---:|:------:|:----:|-------|
| Plan || 🔄 || Plan object model to be completed |
| BasicPlanner |||| |
| ActionPlanner ||| 🔄 | |
| SequentialPlanner ||| 🔄 | |

## Memory Connectors, Vector storage

| | C# | Python | Java | Notes |
|---------------|:---:|:------:|:----:|-------|
| Azure Search || 🔄 || Azure Cognitive Search coming soon |
| Weaviate |||| Currently supported on Python 3.9+, 3.8 coming soon |
| Chroma |||| |
| Qdrant |||| |
| Pinecone |||| |
| Milvus |||| Coming soon |
| Sqlite |||| Vector optimization requires [sqlite-vss](https://github.com/asg017/sqlite-vss) |
| Postgres |||| Vector optimization requires [pgvector](https://github.com/pgvector/pgvector) |
| CosmosDB |||| CosmosDB is not optimized for vector storage |
| Redis |||| Vector optimization requires [RediSearch](https://redis.io/docs/stack/search) |

## Connectors and Skill Libraries

| | C# | Python | Java | Notes |
|---------------------------------------|:---:|:------:|:----:|-------|
| MsGraph |||| Contains connectors for OneDrive, Outlook, ToDos, and Organization Hierarchies |
| Document and Data Loading Skills |||| Pdf, csv, docx, pptx. Currently only supports Word documents |
| OpenAPI |||| |
| Web Search Skills (i.e. Bing, Google) |||| |
| Text Chunkers | 🔄 | 🔄 || |

# Design Choices

The overall architecture of the core kernel is consistent across all languages,
however, the code follows common paradigms and style of each language.

During the initial development phase, many Python best practices have been ignored
in the interest of velocity and feature parity. The project is now going through
a refactoring exercise to increase code quality.

To make the SDK as lightweight as possible, the core packages have
a minimal set of external dependencies.
To make an update on the page, file a PR on the [docs repo.](https://github.com/MicrosoftDocs/semantic-kernel-docs/blob/main/semantic-kernel/get-started/supported-languages.md)
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ Semantic Kernel is available to explore AI and build apps with C# and Python:
</tbody>
</table>

See the [Feature Matrix](FEATURE_MATRIX.md) to see a breakdown of feature parity between C# and Python.
See the [Feature Matrix](https://learn.microsoft.com/en-us/semantic-kernel/get-started/supported-languages) to see a breakdown of feature parity between our currently supported languages.

The quickest way to get started with the basics is to get an API key
(OpenAI or Azure OpenAI)
Expand Down
6 changes: 5 additions & 1 deletion dotnet/Directory.Packages.props
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,15 @@
<ItemGroup>
<PackageVersion Include="Azure.AI.OpenAI" Version="1.0.0-beta.5" />
<PackageVersion Include="Azure.Identity" Version="1.9.0" />
<PackageVersion Include="Azure.Search.Documents" Version="11.5.0-beta.2" />
<PackageVersion Include="Azure.Search.Documents" Version="11.5.0-beta.3" />
<PackageVersion Include="Microsoft.ApplicationInsights.WorkerService" Version="2.21.0" />
<PackageVersion Include="Microsoft.Bcl.HashCode" Version="[1.1.0, )" />
<PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="6.0.0" />
<PackageVersion Include="Microsoft.Extensions.Http" Version="6.0.0" />
<PackageVersion Include="NRedisStack" Version="0.8.0" />
<PackageVersion Include="Pgvector" Version="0.1.3" />
<PackageVersion Include="Polly" Version="7.2.4" />
<PackageVersion Include="System.Diagnostics.DiagnosticSource" Version="7.0.2" />
<PackageVersion Include="System.Linq.Async" Version="6.0.1" />
<PackageVersion Include="System.Text.Json" Version="6.0.8" />
<!-- Microsoft.Extensions.Logging -->
Expand All @@ -29,6 +31,8 @@
<!-- Test -->
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.6.3" />
<PackageVersion Include="Moq" Version="4.18.4" />
<PackageVersion Include="System.Threading.Channels" Version="7.0.0" />
<PackageVersion Include="System.Threading.Tasks.Dataflow" Version="7.0.0" />
<PackageVersion Include="xunit" Version="2.4.2" />
<PackageVersion Include="xunit.runner.visualstudio" Version="2.5.0" />
<PackageVersion Include="coverlet.collector" Version="6.0.0" />
Expand Down
19 changes: 19 additions & 0 deletions dotnet/SK-dotnet.sln
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
Directory.Packages.props = Directory.Packages.props
..\.github\workflows\dotnet-format.yml = ..\.github\workflows\dotnet-format.yml
..\README.md = ..\README.md
..\nuget.config = ..\nuget.config
EndProjectSection
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SemanticKernel.UnitTests", "src\SemanticKernel.UnitTests\SemanticKernel.UnitTests.csproj", "{37E39C68-5A40-4E63-9D3C-0C66AD98DFCB}"
Expand Down Expand Up @@ -140,8 +141,12 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Skills.Core", "src\Skills\S
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NCalcSkills", "samples\NCalcSkills\NCalcSkills.csproj", "{E6EDAB8F-3406-4DBF-9AAB-DF40DC2CA0FA}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.AI.Oobabooga", "src\Connectors\Connectors.AI.Oobabooga\Connectors.AI.Oobabooga.csproj", "{677F1381-7830-4115-9C1A-58B282629DC6}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Planning.StepwisePlanner", "src\Extensions\Planning.StepwisePlanner\Planning.StepwisePlanner.csproj", "{4762BCAF-E1C5-4714-B88D-E50FA333C50E}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ApplicationInsightsExample", "samples\ApplicationInsightsExample\ApplicationInsightsExample.csproj", "{C754950A-E16C-4F96-9CC7-9328E361B5AF}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -341,12 +346,24 @@ Global
{E6EDAB8F-3406-4DBF-9AAB-DF40DC2CA0FA}.Publish|Any CPU.ActiveCfg = Release|Any CPU
{E6EDAB8F-3406-4DBF-9AAB-DF40DC2CA0FA}.Release|Any CPU.ActiveCfg = Release|Any CPU
{E6EDAB8F-3406-4DBF-9AAB-DF40DC2CA0FA}.Release|Any CPU.Build.0 = Release|Any CPU
{677F1381-7830-4115-9C1A-58B282629DC6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{677F1381-7830-4115-9C1A-58B282629DC6}.Debug|Any CPU.Build.0 = Debug|Any CPU
{677F1381-7830-4115-9C1A-58B282629DC6}.Publish|Any CPU.ActiveCfg = Publish|Any CPU
{677F1381-7830-4115-9C1A-58B282629DC6}.Publish|Any CPU.Build.0 = Publish|Any CPU
{677F1381-7830-4115-9C1A-58B282629DC6}.Release|Any CPU.ActiveCfg = Release|Any CPU
{677F1381-7830-4115-9C1A-58B282629DC6}.Release|Any CPU.Build.0 = Release|Any CPU
{4762BCAF-E1C5-4714-B88D-E50FA333C50E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{4762BCAF-E1C5-4714-B88D-E50FA333C50E}.Debug|Any CPU.Build.0 = Debug|Any CPU
{4762BCAF-E1C5-4714-B88D-E50FA333C50E}.Publish|Any CPU.ActiveCfg = Publish|Any CPU
{4762BCAF-E1C5-4714-B88D-E50FA333C50E}.Publish|Any CPU.Build.0 = Publish|Any CPU
{4762BCAF-E1C5-4714-B88D-E50FA333C50E}.Release|Any CPU.ActiveCfg = Release|Any CPU
{4762BCAF-E1C5-4714-B88D-E50FA333C50E}.Release|Any CPU.Build.0 = Release|Any CPU
{C754950A-E16C-4F96-9CC7-9328E361B5AF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{C754950A-E16C-4F96-9CC7-9328E361B5AF}.Debug|Any CPU.Build.0 = Debug|Any CPU
{C754950A-E16C-4F96-9CC7-9328E361B5AF}.Publish|Any CPU.ActiveCfg = Debug|Any CPU
{C754950A-E16C-4F96-9CC7-9328E361B5AF}.Publish|Any CPU.Build.0 = Debug|Any CPU
{C754950A-E16C-4F96-9CC7-9328E361B5AF}.Release|Any CPU.ActiveCfg = Release|Any CPU
{C754950A-E16C-4F96-9CC7-9328E361B5AF}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down Expand Up @@ -396,7 +413,9 @@ Global
{1C19D805-3573-4477-BF07-40180FCDE1BD} = {958AD708-F048-4FAF-94ED-D2F2B92748B9}
{0D0C4DAD-E6BC-4504-AE3A-EEA4E35920C1} = {9ECD1AA0-75B3-4E25-B0B5-9F0945B64974}
{E6EDAB8F-3406-4DBF-9AAB-DF40DC2CA0FA} = {FA3720F1-C99A-49B2-9577-A940257098BF}
{677F1381-7830-4115-9C1A-58B282629DC6} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C}
{4762BCAF-E1C5-4714-B88D-E50FA333C50E} = {078F96B4-09E1-4E0E-B214-F71A4F4BF633}
{C754950A-E16C-4F96-9CC7-9328E361B5AF} = {FA3720F1-C99A-49B2-9577-A940257098BF}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {FBDC56A3-86AD-4323-AA0F-201E59123B83}
Expand Down
2 changes: 2 additions & 0 deletions dotnet/SK-dotnet.sln.DotSettings
Original file line number Diff line number Diff line change
Expand Up @@ -202,8 +202,10 @@ public void It$SOMENAME$()
<s:Boolean x:Key="/Default/UserDictionary/Words/=Kitto/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=lessthan/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=mergeresults/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Mirostat/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=myfile/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Notegen/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Oobabooga/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=pgvector/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Pinecone/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Pinecone_0027s/@EntryIndexedValue">True</s:Boolean>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<RollForward>LatestMajor</RollForward>
<OutputType>Exe</OutputType>
<LangVersion>10</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>disable</ImplicitUsings>
<IsPackable>false</IsPackable>
<!-- Suppress: "Declare types in namespaces", "Require ConfigureAwait" -->
<NoWarn>CA1050;CA1707;CA2007;VSTHRD111</NoWarn>
<UserSecretsId>5ee045b0-aea3-4f08-8d31-32d1a6f8fed0</UserSecretsId>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Microsoft.ApplicationInsights.WorkerService" />
<PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" />
</ItemGroup>

<ItemGroup>
<ProjectReference Include="..\..\src\Extensions\Planning.SequentialPlanner\Planning.SequentialPlanner.csproj" />
<ProjectReference Include="..\..\src\Connectors\Connectors.AI.OpenAI\Connectors.AI.OpenAI.csproj" />
<ProjectReference Include="..\..\src\SemanticKernel\SemanticKernel.csproj" />
</ItemGroup>

</Project>
Loading

0 comments on commit 8b68683

Please sign in to comment.