Python: Add kernel plugin collection and remove old plugin collection…

… classes (#4764)

### Motivation and Context

This PR solves two works items: #4560 and #4667. The Python SDK had
different ways of handling plugins and collections. The work in the PR
helps align the SDK with dotnet nomenclature and structure.

<!-- Thank you for your contribution to the semantic-kernel repo!
Please help reviewers and future users, providing the following
information:
  1. Why is this change required?
  2. What problem does it solve?
  3. What scenario does it contribute to?
  4. If it fixes an open issue, please link to the issue here.
-->

### Description

A `KernelPluginCollection` is introduced with an underlying
`DefaultKernelPlugin` and a `KernelPlugin` base class. Other code is
refactored to use this new `KernelPluginCollection` model which should
simplify how plugins are handled. Unit tests are added to test the new
functionality. Existing unit tests and integration tests, along with
kernel examples and notebooks were updated based on these changes.

Unlike before, we're requiring plugins to be defined with a name, which
removes the GLOBAL_PLUGIN constant name of "__GLOBAL_PLUGIN__". In some
senses, while creating a plugin or a function, a name may not be
provided and so a random name in the form of a plugin
(p_<random_ascii_chars>) or function (f_<random_ascii_chars>) is used.

<!-- Describe your changes, the overall approach, the underlying design.
These notes will help understanding how your code works. Thanks! -->

### Contribution Checklist

<!-- Before submitting this PR, please make sure: -->

- [X] The code builds clean without any errors or warnings
- [X] The PR follows the [SK Contribution
Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md)
and the [pre-submission formatting
script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts)
raises no violations
- [X] All unit tests pass, and I have added new tests where possible
- [ ] I didn't break anyone 😄

---------

Co-authored-by: Evan Mattson <[email protected]>

python/notebooks/05-using-the-planner.ipynb

@@ -669,7 +669,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.7"
+   "version": "3.10.12"
   }
  },
  "nbformat": 4,

python/notebooks/08-native-function-inline.ipynb

@@ -169,7 +169,7 @@
     "    top_p=0.5,\n",
     ")\n",
     "\n",
-    "generate_number_plugin = kernel.import_plugin(GenerateNumberPlugin())"
+    "generate_number_plugin = kernel.import_plugin(GenerateNumberPlugin(), \"GenerateNumberPlugin\")"
    ]
   },
   {
@@ -309,7 +309,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "generate_number_plugin = kernel.import_plugin(GenerateNumberPlugin())\n",
+    "generate_number_plugin = kernel.import_plugin(GenerateNumberPlugin(), \"GenerateNumberPlugin\")\n",
     "generate_number = generate_number_plugin[\"GenerateNumber\"]"
    ]
   },
@@ -496,24 +496,6 @@
     "\"\"\""
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "73aca517",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "corgi_story = kernel.create_semantic_function(\n",
-    "    prompt_template=sk_prompt,\n",
-    "    function_name=\"CorgiStory\",\n",
-    "    plugin_name=\"CorgiPlugin\",\n",
-    "    description=\"Write a short story about two Corgis on an adventure\",\n",
-    "    max_tokens=500,\n",
-    "    temperature=0.5,\n",
-    "    top_p=0.5,\n",
-    ")"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,

python/notebooks/third_party/weaviate-persistent-memory.ipynb

@@ -509,4 +509,4 @@
   },
   "nbformat": 4,
   "nbformat_minor": 2
-}
+}

python/samples/kernel-syntax-examples/google_palm_chat_with_memory.py

@@ -13,7 +13,7 @@
 palm_chat_completion = sk_gp.GooglePalmChatCompletion("models/chat-bison-001", apikey)
 kernel.add_chat_service("models/chat-bison-001", palm_chat_completion)
 kernel.register_memory_store(memory_store=sk.memory.VolatileMemoryStore())
-kernel.import_plugin(sk.core_plugins.TextMemoryPlugin())
+kernel.import_plugin(sk.core_plugins.TextMemoryPlugin(), "TextMemoryPlugin")
 
 
 async def populate_memory(kernel: sk.Kernel) -> None:

python/samples/kernel-syntax-examples/memory.py

@@ -61,7 +61,7 @@ async def setup_chat_with_memory(
     context["fact5"] = "what do I do for work?"
 
     context[sk.core_plugins.TextMemoryPlugin.COLLECTION_PARAM] = "aboutMe"
-    context[sk.core_plugins.TextMemoryPlugin.RELEVANCE_PARAM] = 0.8
+    context[sk.core_plugins.TextMemoryPlugin.RELEVANCE_PARAM] = "0.8"
 
     context["chat_history"] = ""
 
@@ -100,7 +100,7 @@ async def main() -> None:
     )
 
     kernel.register_memory_store(memory_store=sk.memory.VolatileMemoryStore())
-    kernel.import_plugin(sk.core_plugins.TextMemoryPlugin())
+    kernel.import_plugin(sk.core_plugins.TextMemoryPlugin(), "TextMemoryPlugin")
 
     print("Populating memory...")
     await populate_memory(kernel)

python/samples/kernel-syntax-examples/plugins_from_dir.py

@@ -8,7 +8,7 @@
 
 kernel = sk.Kernel()
 
-useAzureOpenAI = True
+useAzureOpenAI = False
 model = "gpt-35-turbo-instruct" if useAzureOpenAI else "gpt-3.5-turbo-instruct"
 service_id = model
 
@@ -23,7 +23,7 @@
     api_key, org_id = sk.openai_settings_from_dot_env()
     kernel.add_text_completion_service(
         service_id,
-        sk_oai.OpenAITextCompletion(deployment_name=model, api_key=api_key, org_id=org_id),
+        sk_oai.OpenAITextCompletion(ai_model_id=model, api_key=api_key, org_id=org_id),
     )
 
 # note: using plugins from the samples folder

python/samples/kernel-syntax-examples/self-critique_rag.py

@@ -48,6 +48,8 @@ async def main() -> None:
     kernel.add_text_completion_service(
         "dv",
         AzureTextCompletion(
+            # Note: text-davinci-003 is deprecated and will be replaced by
+            # AzureOpenAI's gpt-35-turbo-instruct model.
             deployment_name="gpt-35-turbo-instruct",
             endpoint=AZURE_OPENAI_ENDPOINT,
             api_key=AZURE_OPENAI_API_KEY,