Skip to content

Commit

Permalink
feat: add aws-llama-index-data-loader architecture update
Browse files Browse the repository at this point in the history
  • Loading branch information
@scottschreckengaust
scottschreckengaust committed Oct 11, 2024
1 parent 480aadc commit 7646676
Show file tree
Hide file tree
Showing 2 changed files with 21 additions and 18 deletions.
39 changes: 21 additions & 18 deletions docs/generative_ai_cdk_constructs.drawio
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1017,57 +1017,60 @@
<root>
<mxCell id="0" />
<mxCell id="1" parent="0" />
<mxCell id="aCBm29haeZ8VaGctfYJT-1" value="SQS Queue&lt;div&gt;processing&lt;/div&gt;" style="sketch=0;outlineConnect=0;fontColor=#232F3E;gradientColor=none;fillColor=#E7157B;strokeColor=none;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;fontSize=12;fontStyle=0;aspect=fixed;pointerEvents=1;shape=mxgraph.aws4.queue;" vertex="1" parent="1">
<mxCell id="aCBm29haeZ8VaGctfYJT-1" value="SQS Queue&lt;div&gt;processing&lt;/div&gt;" style="sketch=0;outlineConnect=0;fontColor=#232F3E;gradientColor=none;fillColor=#E7157B;strokeColor=none;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;fontSize=12;fontStyle=0;aspect=fixed;pointerEvents=1;shape=mxgraph.aws4.queue;" parent="1" vertex="1">
<mxGeometry x="375" y="176" width="78" height="47" as="geometry" />
</mxCell>
<mxCell id="aCBm29haeZ8VaGctfYJT-2" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="1" source="aCBm29haeZ8VaGctfYJT-3" target="aCBm29haeZ8VaGctfYJT-11">
<mxCell id="aCBm29haeZ8VaGctfYJT-2" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="1" source="aCBm29haeZ8VaGctfYJT-3" target="aCBm29haeZ8VaGctfYJT-11" edge="1">
<mxGeometry relative="1" as="geometry">
<mxPoint x="85.22055917690614" y="200" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="aCBm29haeZ8VaGctfYJT-3" value="S3 Bucket&lt;div&gt;raw&lt;/div&gt;" style="sketch=0;outlineConnect=0;fontColor=#232F3E;gradientColor=none;fillColor=#7AA116;strokeColor=none;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;fontSize=12;fontStyle=0;aspect=fixed;pointerEvents=1;shape=mxgraph.aws4.bucket_with_objects;" vertex="1" parent="1">
<mxCell id="aCBm29haeZ8VaGctfYJT-3" value="S3 Bucket&lt;div&gt;raw&lt;/div&gt;" style="sketch=0;outlineConnect=0;fontColor=#232F3E;gradientColor=none;fillColor=#7AA116;strokeColor=none;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;fontSize=12;fontStyle=0;aspect=fixed;pointerEvents=1;shape=mxgraph.aws4.bucket_with_objects;" parent="1" vertex="1">
<mxGeometry x="65" y="161" width="75" height="78" as="geometry" />
</mxCell>
<mxCell id="aCBm29haeZ8VaGctfYJT-4" value="Auto Scaling group&lt;div&gt;Loader&lt;/div&gt;" style="points=[[0,0],[0.25,0],[0.5,0],[0.75,0],[1,0],[1,0.25],[1,0.5],[1,0.75],[1,1],[0.75,1],[0.5,1],[0.25,1],[0,1],[0,0.75],[0,0.5],[0,0.25]];outlineConnect=0;gradientColor=none;html=1;whiteSpace=wrap;fontSize=12;fontStyle=0;container=1;pointerEvents=0;collapsible=0;recursiveResize=0;shape=mxgraph.aws4.groupCenter;grIcon=mxgraph.aws4.group_auto_scaling_group;grStroke=1;strokeColor=#D86613;fillColor=none;verticalAlign=top;align=center;fontColor=#D86613;dashed=1;spacingTop=25;" vertex="1" parent="1">
<mxCell id="aCBm29haeZ8VaGctfYJT-4" value="Auto Scaling group&lt;div&gt;Loader&lt;/div&gt;" style="points=[[0,0],[0.25,0],[0.5,0],[0.75,0],[1,0],[1,0.25],[1,0.5],[1,0.75],[1,1],[0.75,1],[0.5,1],[0.25,1],[0,1],[0,0.75],[0,0.5],[0,0.25]];outlineConnect=0;gradientColor=none;html=1;whiteSpace=wrap;fontSize=12;fontStyle=0;container=1;pointerEvents=0;collapsible=0;recursiveResize=0;shape=mxgraph.aws4.groupCenter;grIcon=mxgraph.aws4.group_auto_scaling_group;grStroke=1;strokeColor=#D86613;fillColor=none;verticalAlign=top;align=center;fontColor=#D86613;dashed=1;spacingTop=25;" parent="1" vertex="1">
<mxGeometry x="564" y="110" width="130" height="180" as="geometry" />
</mxCell>
<mxCell id="aCBm29haeZ8VaGctfYJT-5" value="ECS Task&lt;div&gt;reader&lt;/div&gt;" style="sketch=0;outlineConnect=0;fontColor=#232F3E;gradientColor=none;fillColor=#ED7100;strokeColor=none;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;fontSize=12;fontStyle=0;aspect=fixed;pointerEvents=1;shape=mxgraph.aws4.ecs_task;" vertex="1" parent="aCBm29haeZ8VaGctfYJT-4">
<mxCell id="aCBm29haeZ8VaGctfYJT-5" value="ECS Task&lt;div&gt;reader&lt;/div&gt;" style="sketch=0;outlineConnect=0;fontColor=#232F3E;gradientColor=none;fillColor=#ED7100;strokeColor=none;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;fontSize=12;fontStyle=0;aspect=fixed;pointerEvents=1;shape=mxgraph.aws4.ecs_task;" parent="aCBm29haeZ8VaGctfYJT-4" vertex="1">
<mxGeometry x="46.5" y="66" width="37" height="48" as="geometry" />
</mxCell>
<mxCell id="aCBm29haeZ8VaGctfYJT-6" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;" edge="1" parent="1" source="aCBm29haeZ8VaGctfYJT-4" target="aCBm29haeZ8VaGctfYJT-4">
<mxCell id="aCBm29haeZ8VaGctfYJT-6" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;" parent="1" source="aCBm29haeZ8VaGctfYJT-4" target="aCBm29haeZ8VaGctfYJT-4" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="aCBm29haeZ8VaGctfYJT-7" value="long poller" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="1" source="aCBm29haeZ8VaGctfYJT-5" target="aCBm29haeZ8VaGctfYJT-1">
<mxCell id="aCBm29haeZ8VaGctfYJT-7" value="long poller" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="1" source="aCBm29haeZ8VaGctfYJT-5" target="aCBm29haeZ8VaGctfYJT-1" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="aCBm29haeZ8VaGctfYJT-8" value="S3 Bucket&lt;div&gt;text&amp;nbsp;&lt;/div&gt;&lt;div&gt;(optional sibling metadata)&lt;/div&gt;" style="sketch=0;outlineConnect=0;fontColor=#232F3E;gradientColor=none;fillColor=#7AA116;strokeColor=none;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;fontSize=12;fontStyle=0;aspect=fixed;pointerEvents=1;shape=mxgraph.aws4.bucket_with_objects;" vertex="1" parent="1">
<mxCell id="aCBm29haeZ8VaGctfYJT-8" value="S3 Bucket&lt;div&gt;text&amp;nbsp;&lt;/div&gt;&lt;div&gt;(optional sibling metadata)&lt;/div&gt;" style="sketch=0;outlineConnect=0;fontColor=#232F3E;gradientColor=none;fillColor=#7AA116;strokeColor=none;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;fontSize=12;fontStyle=0;aspect=fixed;pointerEvents=1;shape=mxgraph.aws4.bucket_with_objects;" parent="1" vertex="1">
<mxGeometry x="775" y="161" width="75" height="78" as="geometry" />
</mxCell>
<mxCell id="aCBm29haeZ8VaGctfYJT-9" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="1" source="aCBm29haeZ8VaGctfYJT-5" target="aCBm29haeZ8VaGctfYJT-8">
<mxCell id="aCBm29haeZ8VaGctfYJT-9" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="1" source="aCBm29haeZ8VaGctfYJT-5" target="aCBm29haeZ8VaGctfYJT-8" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="aCBm29haeZ8VaGctfYJT-10" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="1" source="aCBm29haeZ8VaGctfYJT-11" target="aCBm29haeZ8VaGctfYJT-1">
<mxCell id="aCBm29haeZ8VaGctfYJT-10" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="1" source="aCBm29haeZ8VaGctfYJT-11" target="aCBm29haeZ8VaGctfYJT-1" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="aCBm29haeZ8VaGctfYJT-11" value="SNS Topic&lt;div&gt;object event&lt;/div&gt;" style="sketch=0;outlineConnect=0;fontColor=#232F3E;gradientColor=none;fillColor=#E7157B;strokeColor=none;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;fontSize=12;fontStyle=0;aspect=fixed;pointerEvents=1;shape=mxgraph.aws4.topic;" vertex="1" parent="1">
<mxCell id="aCBm29haeZ8VaGctfYJT-11" value="SNS Topic&lt;div&gt;object event&lt;/div&gt;" style="sketch=0;outlineConnect=0;fontColor=#232F3E;gradientColor=none;fillColor=#E7157B;strokeColor=none;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;fontSize=12;fontStyle=0;aspect=fixed;pointerEvents=1;shape=mxgraph.aws4.topic;" parent="1" vertex="1">
<mxGeometry x="215" y="166.5" width="78" height="67" as="geometry" />
</mxCell>
<mxCell id="aCBm29haeZ8VaGctfYJT-12" value="AWS Cloud" style="points=[[0,0],[0.25,0],[0.5,0],[0.75,0],[1,0],[1,0.25],[1,0.5],[1,0.75],[1,1],[0.75,1],[0.5,1],[0.25,1],[0,1],[0,0.75],[0,0.5],[0,0.25]];outlineConnect=0;gradientColor=none;html=1;whiteSpace=wrap;fontSize=12;fontStyle=0;container=1;pointerEvents=0;collapsible=0;recursiveResize=0;shape=mxgraph.aws4.group;grIcon=mxgraph.aws4.group_aws_cloud_alt;strokeColor=#232F3E;fillColor=none;verticalAlign=top;align=left;spacingLeft=30;fontColor=#232F3E;dashed=0;" vertex="1" parent="1">
<mxCell id="aCBm29haeZ8VaGctfYJT-12" value="AWS Cloud" style="points=[[0,0],[0.25,0],[0.5,0],[0.75,0],[1,0],[1,0.25],[1,0.5],[1,0.75],[1,1],[0.75,1],[0.5,1],[0.25,1],[0,1],[0,0.75],[0,0.5],[0,0.25]];outlineConnect=0;gradientColor=none;html=1;whiteSpace=wrap;fontSize=12;fontStyle=0;container=1;pointerEvents=0;collapsible=0;recursiveResize=0;shape=mxgraph.aws4.group;grIcon=mxgraph.aws4.group_aws_cloud_alt;strokeColor=#232F3E;fillColor=none;verticalAlign=top;align=left;spacingLeft=30;fontColor=#232F3E;dashed=0;" parent="1" vertex="1">
<mxGeometry x="10" y="10" width="950" height="490" as="geometry" />
</mxCell>
<mxCell id="aCBm29haeZ8VaGctfYJT-13" value="Region" style="points=[[0,0],[0.25,0],[0.5,0],[0.75,0],[1,0],[1,0.25],[1,0.5],[1,0.75],[1,1],[0.75,1],[0.5,1],[0.25,1],[0,1],[0,0.75],[0,0.5],[0,0.25]];outlineConnect=0;gradientColor=none;html=1;whiteSpace=wrap;fontSize=12;fontStyle=0;container=1;pointerEvents=0;collapsible=0;recursiveResize=0;shape=mxgraph.aws4.group;grIcon=mxgraph.aws4.group_region;strokeColor=#00A4A6;fillColor=none;verticalAlign=top;align=left;spacingLeft=30;fontColor=#147EBA;dashed=1;" vertex="1" parent="1">
<mxCell id="aCBm29haeZ8VaGctfYJT-13" value="Region" style="points=[[0,0],[0.25,0],[0.5,0],[0.75,0],[1,0],[1,0.25],[1,0.5],[1,0.75],[1,1],[0.75,1],[0.5,1],[0.25,1],[0,1],[0,0.75],[0,0.5],[0,0.25]];outlineConnect=0;gradientColor=none;html=1;whiteSpace=wrap;fontSize=12;fontStyle=0;container=1;pointerEvents=0;collapsible=0;recursiveResize=0;shape=mxgraph.aws4.group;grIcon=mxgraph.aws4.group_region;strokeColor=#00A4A6;fillColor=none;verticalAlign=top;align=left;spacingLeft=30;fontColor=#147EBA;dashed=1;" parent="1" vertex="1">
<mxGeometry x="30" y="50" width="890" height="430" as="geometry" />
</mxCell>
<mxCell id="aCBm29haeZ8VaGctfYJT-14" value="ECS Container&lt;div&gt;llamaindex code&lt;/div&gt;" style="sketch=0;outlineConnect=0;fontColor=#232F3E;gradientColor=none;fillColor=#ED7100;strokeColor=none;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;fontSize=12;fontStyle=0;aspect=fixed;pointerEvents=1;shape=mxgraph.aws4.container_1;" vertex="1" parent="aCBm29haeZ8VaGctfYJT-13">
<mxCell id="aCBm29haeZ8VaGctfYJT-14" value="ECS Container&lt;div&gt;llamaindex code&lt;/div&gt;" style="sketch=0;outlineConnect=0;fontColor=#232F3E;gradientColor=none;fillColor=#ED7100;strokeColor=none;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;fontSize=12;fontStyle=0;aspect=fixed;pointerEvents=1;shape=mxgraph.aws4.container_1;" parent="aCBm29haeZ8VaGctfYJT-13" vertex="1">
<mxGeometry x="575" y="337" width="48" height="31" as="geometry" />
</mxCell>
<mxCell id="aCBm29haeZ8VaGctfYJT-15" value="SSM Parameter Store&lt;div&gt;circuit breaker&lt;/div&gt;" style="sketch=0;outlineConnect=0;fontColor=#232F3E;gradientColor=none;fillColor=#E7157B;strokeColor=none;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;fontSize=12;fontStyle=0;aspect=fixed;pointerEvents=1;shape=mxgraph.aws4.parameter_store;" vertex="1" parent="aCBm29haeZ8VaGctfYJT-13">
<mxGeometry x="350" y="276" width="75" height="78" as="geometry" />
<mxCell id="aCBm29haeZ8VaGctfYJT-15" value="SSM Parameter Store&lt;div&gt;circuit breaker&lt;/div&gt;" style="sketch=0;outlineConnect=0;fontColor=#232F3E;gradientColor=none;fillColor=#E7157B;strokeColor=none;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;fontSize=12;fontStyle=0;aspect=fixed;pointerEvents=1;shape=mxgraph.aws4.parameter_store;" parent="aCBm29haeZ8VaGctfYJT-13" vertex="1">
<mxGeometry x="360" y="276" width="75" height="78" as="geometry" />
</mxCell>
<mxCell id="aCBm29haeZ8VaGctfYJT-16" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=1;entryDx=0;entryDy=0;" edge="1" parent="1" source="aCBm29haeZ8VaGctfYJT-14" target="aCBm29haeZ8VaGctfYJT-4">
<mxCell id="Of24hmTfBcJFFYMwheT0-1" value="LllamaIndexDataLoader" style="fillColor=none;strokeColor=#5A6C86;dashed=1;verticalAlign=top;fontStyle=0;fontColor=#5A6C86;whiteSpace=wrap;html=1;" vertex="1" parent="aCBm29haeZ8VaGctfYJT-13">
<mxGeometry x="310" y="20" width="560" height="400" as="geometry" />
</mxCell>
<mxCell id="aCBm29haeZ8VaGctfYJT-16" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=1;entryDx=0;entryDy=0;" parent="1" source="aCBm29haeZ8VaGctfYJT-14" target="aCBm29haeZ8VaGctfYJT-4" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="aCBm29haeZ8VaGctfYJT-17" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0;entryY=0.75;entryDx=0;entryDy=0;" edge="1" parent="1" source="aCBm29haeZ8VaGctfYJT-15" target="aCBm29haeZ8VaGctfYJT-4">
<mxCell id="aCBm29haeZ8VaGctfYJT-17" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0;entryY=0.75;entryDx=0;entryDy=0;" parent="1" source="aCBm29haeZ8VaGctfYJT-15" target="aCBm29haeZ8VaGctfYJT-4" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
</root>
Expand Down
Binary file modified src/patterns/gen-ai/aws-llama-index-data-loader/architecture.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.

0 comments on commit 7646676

Please sign in to comment.