From d06af9435a1fb85091fc540c24d20b3529d67d41 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Tue, 15 Oct 2024 18:18:46 -0700 Subject: [PATCH 1/7] Publish llama-index plugin --- .changeset/shiny-emus-change.md | 5 +++ .github/workflows/build-package.yml | 13 +++--- .github/workflows/check-types.yml | 4 +- .github/workflows/publish-docs.yml | 11 +++-- .../livekit-plugins-llama-index/README.md | 40 ++++++++++++++----- 5 files changed, 52 insertions(+), 21 deletions(-) create mode 100644 .changeset/shiny-emus-change.md diff --git a/.changeset/shiny-emus-change.md b/.changeset/shiny-emus-change.md new file mode 100644 index 000000000..8e2423c2e --- /dev/null +++ b/.changeset/shiny-emus-change.md @@ -0,0 +1,5 @@ +--- +"livekit-plugins-llama-index": patch +--- + +Publish llama-index plugin diff --git a/.github/workflows/build-package.yml b/.github/workflows/build-package.yml index 129e794bf..5be08fc1d 100644 --- a/.github/workflows/build-package.yml +++ b/.github/workflows/build-package.yml @@ -12,13 +12,13 @@ on: workflow_dispatch: inputs: package: - description: 'Name of the package to build' + description: "Name of the package to build" required: true - default: 'livekit-plugins-browser' + default: "livekit-plugins-browser" artifact_name: - description: 'Artifact name for the distribution package' + description: "Artifact name for the distribution package" required: true - default: 'build-artifact' + default: "build-artifact" jobs: build_plugins: @@ -35,7 +35,8 @@ jobs: inputs.package == 'livekit-plugins-openai' || inputs.package == 'livekit-plugins-rag' || inputs.package == 'livekit-plugins-silero' || - inputs.package == 'livekit-plugins-anthropic' + inputs.package == 'livekit-plugins-anthropic' || + inputs.package == 'livekit-plugins-llama-index' defaults: run: @@ -98,4 +99,4 @@ jobs: uses: actions/upload-artifact@v3 with: name: ${{ inputs.artifact_name }} - path: livekit-plugins/livekit-plugins-browser/dist/ \ No newline at end of file + path: livekit-plugins/livekit-plugins-browser/dist/ diff --git a/.github/workflows/check-types.yml b/.github/workflows/check-types.yml index 927c9e2eb..9bc413831 100644 --- a/.github/workflows/check-types.yml +++ b/.github/workflows/check-types.yml @@ -20,7 +20,6 @@ jobs: with: submodules: recursive - - uses: actions/setup-python@v5 with: python-version: "3.9" @@ -41,7 +40,8 @@ jobs: ./livekit-plugins/livekit-plugins-cartesia \ ./livekit-plugins/livekit-plugins-rag \ ./livekit-plugins/livekit-plugins-azure \ - ./livekit-plugins/livekit-plugins-anthropic + ./livekit-plugins/livekit-plugins-anthropic \ + ./livekit-plugins/livekit-plugins-llama-index - name: Install stub packages run: | diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml index 8c5f924bd..d6ced9dc8 100644 --- a/.github/workflows/publish-docs.yml +++ b/.github/workflows/publish-docs.yml @@ -3,7 +3,7 @@ name: Publish docs on: workflow_dispatch: workflow_call: - secrets: + secrets: DOCS_DEPLOY_AWS_ACCESS_KEY: {} DOCS_DEPLOY_AWS_API_SECRET: {} @@ -19,7 +19,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: '3.12' # Specify the Python version you want to use + python-version: "3.12" # Specify the Python version you want to use - name: Create and activate virtual environment run: | @@ -42,7 +42,10 @@ jobs: ./livekit-plugins/livekit-plugins-elevenlabs \ ./livekit-plugins/livekit-plugins-google \ ./livekit-plugins/livekit-plugins-nltk \ - ./livekit-plugins/livekit-plugins-openai + ./livekit-plugins/livekit-plugins-openai \ + ./livekit-plugins/livekit-plugins-rag \ + ./livekit-plugins/livekit-plugins-silero \ + ./livekit-plugins/livekit-plugins-llama-index - name: Build Docs run: | @@ -56,4 +59,4 @@ jobs: env: AWS_ACCESS_KEY_ID: ${{ secrets.DOCS_DEPLOY_AWS_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.DOCS_DEPLOY_AWS_API_SECRET }} - AWS_DEFAULT_REGION: "us-east-1" \ No newline at end of file + AWS_DEFAULT_REGION: "us-east-1" diff --git a/livekit-plugins/livekit-plugins-llama-index/README.md b/livekit-plugins/livekit-plugins-llama-index/README.md index 006906528..50f0cb803 100644 --- a/livekit-plugins/livekit-plugins-llama-index/README.md +++ b/livekit-plugins/livekit-plugins-llama-index/README.md @@ -1,14 +1,36 @@ -# LiveKit Plugins Minimal +# LiveKit Plugins Llama Index -This is a minimal example of a LiveKit plugin for Agents. +Agent Framework plugin for using Llama Index. Currently supports [Query Engine](https://docs.llamaindex.ai/en/stable/module_guides/deploying/query_engine/) and [Chat Engine](https://docs.llamaindex.ai/en/stable/module_guides/deploying/chat_engines/). -### Developer note +## Install -When copying this directory over to create a new `livekit-plugins` package, make sure it's nested within the `livekit-plugins` folder and that the `"name"` field in `package.json` follows the proper naming convention for CI: +```bash +pip install livekit-plugins-llama-index +``` + +## Query Engine + +Query Engine is primarily used for RAG. See [example voice agent](https://github.com/livekit/agents/blob/main/examples/voice-pipeline-agent/llamaindex-rag/query_engine.py) + +## Chat Engine -```json -{ - "name": "livekit-plugins-", - "private": true -} +Chat Engine can be used as an LLM within the frame work. + +```python +# load the existing index +storage_context = StorageContext.from_defaults(persist_dir=) +index = load_index_from_storage(storage_context) +chat_engine = index.as_chat_engine(chat_mode=ChatMode.CONTEXT) + +async def entrypoint(ctx: JobContext): + ... + assistant = VoicePipelineAgent( + vad=silero.VAD.load(), + stt=deepgram.STT(), + llm=llama_index.LLM(chat_engine=chat_engine), + tts=openai.TTS(), + chat_ctx=initial_ctx, + ) ``` + +full example [here](https://github.com/livekit/agents/blob/main/examples/voice-pipeline-agent/llamaindex-rag/chat_engine.py) From 830fa15e46cb6efc925db94b98c083f74cb520ad Mon Sep 17 00:00:00 2001 From: David Zhao Date: Tue, 15 Oct 2024 21:26:57 -0700 Subject: [PATCH 2/7] feedback --- .../livekit-plugins-azure/livekit/plugins/azure/stt.py | 2 ++ livekit-plugins/livekit-plugins-llama-index/README.md | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/livekit-plugins/livekit-plugins-azure/livekit/plugins/azure/stt.py b/livekit-plugins/livekit-plugins-azure/livekit/plugins/azure/stt.py index b3ae6b9ee..a210bde08 100644 --- a/livekit-plugins/livekit-plugins-azure/livekit/plugins/azure/stt.py +++ b/livekit-plugins/livekit-plugins-azure/livekit/plugins/azure/stt.py @@ -173,6 +173,8 @@ def _on_session_stopped(self, evt: speechsdk.SpeechRecognitionEventArgs): self._loop.call_soon_threadsafe(self._done_event.set) def _threadsafe_send(self, evt: stt.SpeechEvent | None): + if evt is None: + return self._loop.call_soon_threadsafe(self._event_ch.send_nowait, evt) diff --git a/livekit-plugins/livekit-plugins-llama-index/README.md b/livekit-plugins/livekit-plugins-llama-index/README.md index 50f0cb803..1c2b3add1 100644 --- a/livekit-plugins/livekit-plugins-llama-index/README.md +++ b/livekit-plugins/livekit-plugins-llama-index/README.md @@ -14,16 +14,16 @@ Query Engine is primarily used for RAG. See [example voice agent](https://github ## Chat Engine -Chat Engine can be used as an LLM within the frame work. +Chat Engine can be used as an LLM within the framework. ```python # load the existing index storage_context = StorageContext.from_defaults(persist_dir=) index = load_index_from_storage(storage_context) -chat_engine = index.as_chat_engine(chat_mode=ChatMode.CONTEXT) async def entrypoint(ctx: JobContext): ... + chat_engine = index.as_chat_engine(chat_mode=ChatMode.CONTEXT) assistant = VoicePipelineAgent( vad=silero.VAD.load(), stt=deepgram.STT(), From 6068050c0a90ccf340b5b11977f6355399893799 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Tue, 15 Oct 2024 21:37:16 -0700 Subject: [PATCH 3/7] fix type check --- .../livekit/agents/pipeline/human_input.py | 12 ++++-------- .../livekit/plugins/azure/stt.py | 4 +--- .../livekit/plugins/google/tts.py | 2 +- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/livekit-agents/livekit/agents/pipeline/human_input.py b/livekit-agents/livekit/agents/pipeline/human_input.py index 10342662b..11a3112f8 100644 --- a/livekit-agents/livekit/agents/pipeline/human_input.py +++ b/livekit-agents/livekit/agents/pipeline/human_input.py @@ -80,18 +80,14 @@ def _subscribe_to_microphone(self, *args, **kwargs) -> None: if not publication.subscribed: publication.set_subscribed(True) - if ( - publication.track is not None - and publication.track != self._subscribed_track - ): - self._subscribed_track = publication.track # type: ignore + track = publication.track + if track is not None and track != self._subscribed_track: + self._subscribed_track = track if self._recognize_atask is not None: self._recognize_atask.cancel() self._recognize_atask = asyncio.create_task( - self._recognize_task( - rtc.AudioStream(self._subscribed_track, sample_rate=16000) - ) # type: ignore + self._recognize_task(rtc.AudioStream(track, sample_rate=16000)) ) break diff --git a/livekit-plugins/livekit-plugins-azure/livekit/plugins/azure/stt.py b/livekit-plugins/livekit-plugins-azure/livekit/plugins/azure/stt.py index a210bde08..d5357b604 100644 --- a/livekit-plugins/livekit-plugins-azure/livekit/plugins/azure/stt.py +++ b/livekit-plugins/livekit-plugins-azure/livekit/plugins/azure/stt.py @@ -172,9 +172,7 @@ def _on_speech_end(self, evt: speechsdk.SpeechRecognitionEventArgs): def _on_session_stopped(self, evt: speechsdk.SpeechRecognitionEventArgs): self._loop.call_soon_threadsafe(self._done_event.set) - def _threadsafe_send(self, evt: stt.SpeechEvent | None): - if evt is None: - return + def _threadsafe_send(self, evt: stt.SpeechEvent): self._loop.call_soon_threadsafe(self._event_ch.send_nowait, evt) diff --git a/livekit-plugins/livekit-plugins-google/livekit/plugins/google/tts.py b/livekit-plugins/livekit-plugins-google/livekit/plugins/google/tts.py index 5d0183d60..9b7fad26d 100644 --- a/livekit-plugins/livekit-plugins-google/livekit/plugins/google/tts.py +++ b/livekit-plugins/livekit-plugins-google/livekit/plugins/google/tts.py @@ -200,7 +200,7 @@ async def _main_task(self) -> None: def _gender_from_str(gender: str) -> SsmlVoiceGender: - ssml_gender = SsmlVoiceGender.NEUTRAL + ssml_gender: SsmlVoiceGender = SsmlVoiceGender.NEUTRAL if gender == "male": ssml_gender = SsmlVoiceGender.MALE elif gender == "female": From 684fb570ddfab99c848500d286b48df3352f4d2e Mon Sep 17 00:00:00 2001 From: David Zhao Date: Tue, 15 Oct 2024 21:45:19 -0700 Subject: [PATCH 4/7] more types --- livekit-agents/livekit/agents/pipeline/human_input.py | 2 +- .../livekit-plugins-google/livekit/plugins/google/tts.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/livekit-agents/livekit/agents/pipeline/human_input.py b/livekit-agents/livekit/agents/pipeline/human_input.py index 11a3112f8..b54ba6f28 100644 --- a/livekit-agents/livekit/agents/pipeline/human_input.py +++ b/livekit-agents/livekit/agents/pipeline/human_input.py @@ -80,7 +80,7 @@ def _subscribe_to_microphone(self, *args, **kwargs) -> None: if not publication.subscribed: publication.set_subscribed(True) - track = publication.track + track: rtc.RemoteAudioTrack | None = publication.track # type: ignore if track is not None and track != self._subscribed_track: self._subscribed_track = track if self._recognize_atask is not None: diff --git a/livekit-plugins/livekit-plugins-google/livekit/plugins/google/tts.py b/livekit-plugins/livekit-plugins-google/livekit/plugins/google/tts.py index 9b7fad26d..99c6b5d94 100644 --- a/livekit-plugins/livekit-plugins-google/livekit/plugins/google/tts.py +++ b/livekit-plugins/livekit-plugins-google/livekit/plugins/google/tts.py @@ -200,10 +200,10 @@ async def _main_task(self) -> None: def _gender_from_str(gender: str) -> SsmlVoiceGender: - ssml_gender: SsmlVoiceGender = SsmlVoiceGender.NEUTRAL + ssml_gender = SsmlVoiceGender.NEUTRAL if gender == "male": ssml_gender = SsmlVoiceGender.MALE elif gender == "female": ssml_gender = SsmlVoiceGender.FEMALE - return ssml_gender + return ssml_gender # type: ignore From d91b1b3af7a2f847ba8cae7b0e69cfd6404c44e4 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Tue, 15 Oct 2024 21:51:04 -0700 Subject: [PATCH 5/7] skip tests on forks --- .github/workflows/tests.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9d6f73da0..cddcbd6b2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,11 +13,8 @@ on: jobs: tests: - if: > # don't run tests for PRs on forks - ${{ - !github.event.pull_request || - github.event.pull_request.head.repo.full_name == github.repository - }} + # don't run tests for PRs on forks + if: github.repository == 'livekit/livekit-agents' strategy: fail-fast: false matrix: @@ -43,8 +40,8 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: '3.9' - cache: 'pip' + python-version: "3.9" + cache: "pip" - name: Install ffmpeg (Linux) if: ${{ matrix.platform == 'linux' }} From 1a8a7afa1d8dae9a2e10fb6aae539c2a9c252b8e Mon Sep 17 00:00:00 2001 From: David Zhao Date: Tue, 15 Oct 2024 21:52:00 -0700 Subject: [PATCH 6/7] works --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cddcbd6b2..2e73a09f2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,7 +14,7 @@ on: jobs: tests: # don't run tests for PRs on forks - if: github.repository == 'livekit/livekit-agents' + if: github.repository == 'livekit/agents' strategy: fail-fast: false matrix: From 69526ae1190458b0fbff9140ca2f7611b86398fd Mon Sep 17 00:00:00 2001 From: David Zhao Date: Tue, 15 Oct 2024 22:04:42 -0700 Subject: [PATCH 7/7] improve test workflow --- .github/workflows/tests.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2e73a09f2..7febfaed6 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -38,7 +38,20 @@ jobs: submodules: true lfs: true + - name: Cache packages + uses: actions/cache@v4 + with: + path: | + /var/cache/apt/archives + ~/Library/Caches/Homebrew + C:\ProgramData\chocolatey\lib\ffmpeg + key: ${{ runner.os }}-cache + restore-keys: | + ${{ runner.os }}-cache + - uses: actions/setup-python@v5 + # on mac, ffmpeg installs python + if: ${{ matrix.os != 'macos' }} with: python-version: "3.9" cache: "pip"