Update 2.x with latest commits on main (opensearch-project#1070)

* Update mocha (opensearch-project#960)
* added code for the change of design of the sidebar, fixed overriding … (opensearch-project#928)
* Refactor add_integration_flyout to break out nested methods (opensearch-project#956)
* [Log Explorer] Remove top level tabs (opensearch-project#970)
* Explorer's timeheader & color change (opensearch-project#959)
* Update caching (opensearch-project#965)
* Move query_utils to /public (opensearch-project#983)
* Remove Zeppelin Parser from Notebook (opensearch-project#985)
* Explorer empty bucket (opensearch-project#990)
* refactored classnames for sidebar (opensearch-project#933)
* Refactor Integrations Backend to abstract IO (opensearch-project#947)
* Create React component for Integrations Setup UI (opensearch-project#1009)
* Create Data sources plugin with Manage Datasources Flow  (opensearch-project#1035)
* Manage datasources (opensearch-project#967)
* Add acceleration management UI  (opensearch-project#989)
* Rename data sources to data connections (opensearch-project#1004)
* Add fallback to show if user does not have datasource API permissions (opensearch-project#1008)
* Add access control tab content (opensearch-project#992)
* Delete datasource and Connection Configuration Tab (opensearch-project#1024)
* adjust explorer chart color and spacing (opensearch-project#1051)
* Configure S3 datasource flow (opensearch-project#1049)
* Manage datasources (opensearch-project#967)
* Add acceleration management UI  (opensearch-project#989)
* Rename data sources to data connections (opensearch-project#1004)
* Add fallback to show if user does not have datasource API permissions (opensearch-project#1008)
* Add access control tab content (opensearch-project#992)
* Delete datasource and Connection Configuration Tab (opensearch-project#1024)
* modified explorer data grid to follow discover look and feel (opensearch-project#1041)
* Create prometheus datasource flow (opensearch-project#1054)
* Manage datasources (opensearch-project#967)
* Add acceleration management UI  (opensearch-project#989)
* Rename data sources to data connections (opensearch-project#1004)
* Add fallback to show if user does not have datasource API permissions (opensearch-project#1008)
* Add access control tab content (opensearch-project#992)
* Delete datasource and Connection Configuration Tab (opensearch-project#1024)
* added design changes for sidebar (opensearch-project#1061)
* updated snapshots (opensearch-project#1062)
* Update UI for integrations setup (opensearch-project#1052)
* Data sources UI improvements (opensearch-project#1059)
* adding redirection to datasources (opensearch-project#1063)
* Move convertDateTime, refactor query_utils (opensearch-project#1064)
* Setup S3 connection with integrations (opensearch-project#1057)
* Use approved svg from UX (opensearch-project#1066)
* Fix missing import 'moment' on query_utils. (opensearch-project#1067)

---------

Signed-off-by: Derek Ho <dxho@amazon.com>
Signed-off-by: sumukhswamy <sumukhhs@amazon.com>
Signed-off-by: Eric Wei <menwe@amazon.com>
Signed-off-by: Paul Sebastian <paulstn@amazon.com>
Signed-off-by: Peter Fitzgibbons <peter.fitzgibbons@gmail.com>
Signed-off-by: Shenoy Pratik <sgguruda@amazon.com>
Signed-off-by: Simeon Widdis <sawiddis@gmail.com>
Co-authored-by: sumukhswamy <sumukhhs@amazon.com>
Co-authored-by: Eric Wei <menwe@amazon.com>
Co-authored-by: Paul Sebastian <paul_sebastian@live.com>
Co-authored-by: Peter Fitzgibbons <peter.fitzgibbons@gmail.com>
Co-authored-by: Paul Sebastian <paulstn@amazon.com>
Co-authored-by: Shenoy Pratik <sgguruda@amazon.com>
Co-authored-by: Simeon Widdis <sawiddis@amazon.com>
(cherry picked from commit 171acde)

.cypress/integration/10_datasources.spec.js

@@ -0,0 +1,42 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/// <reference types="cypress" />
+
+
+  const moveToDatasourcesHome = () => {
+    cy.visit(`${Cypress.env('opensearchDashboards')}/app/datasources`);
+  };
+
+  const moveToNewDatasourcesPage = () => {
+    cy.visit(`${Cypress.env('opensearchDashboards')}/app/datasources#/new`);
+  };
+
+  const moveToCreatePrometheusDatasourcePage = () => {
+    cy.visit(`${Cypress.env('opensearchDashboards')}/app/datasources#/configure/PROMETHEUS`);
+  };
+
+  describe('Integration tests for datasources plugin', () => {
+    const testPrometheusSuffix = (Math.random() + 1).toString(36).substring(7);
+const testPrometheusInstance = `Prometheus_${testPrometheusSuffix}`;
+const testS3Suffix = (Math.random() + 1).toString(36).substring(7);
+const testS3Instance = `S3_${testS3Suffix}`;
+    it('Navigates to datasources plugin and expects the correct header', () => {
+      moveToDatasourcesHome();
+      cy.get('[data-test-subj="dataconnections-header"]').should('exist');
+    });
+
+    it('Tests navigation between tabs and goes to Prometheus creation flow', () => {
+      moveToDatasourcesHome();
+      cy.get('[data-test-subj="new"]').click();
+      cy.url().should('include', '/new')
+      cy.get('[data-test-subj="datasource_card_prometheus"]').click();
+      cy.url().should('include', '/configure/PROMETHEUS');
+    });
+
+});
+
+
+

.cypress/integration/1_event_analytics.spec.js

@@ -108,7 +108,8 @@ describe('Open flyout for a data row to see details', () => {
   });
 });
 
-describe('Add/delete/switch explorer top level tabs', () => {
+// skip for now due to tab removals
+describe.skip('Add/delete/switch explorer top level tabs', () => {
   beforeEach(() => {
     landOnEventExplorer();
   });
@@ -406,15 +407,15 @@ describe('Live tail stop automatically', () => {
     landOnEventExplorer();
   });
 
-  it('Moving to other tab should stop live tail automatically', () => {
+  it.skip('Moving to other tab should stop live tail automatically', () => {
     clearQuerySearchBoxText('searchAutocompleteTextArea');
     cy.get('[data-test-subj="searchAutocompleteTextArea"]').type(TEST_QUERIES[1].query);
     cy.get('[data-test-subj=eventLiveTail]').click();
     cy.get('[data-test-subj=eventLiveTail__delay10s]').click();
     cy.get('.euiToastHeader__title').contains('On').should('exist');
   });
 
-  it('Add a new tab', () => {
+  it.skip('Add a new tab', () => {
     cy.get('[data-test-subj="eventExplorer__topLevelTabbing"]')
       .find('button.euiTab')
       .then((lists) => {
@@ -426,7 +427,7 @@ describe('Live tail stop automatically', () => {
       });
   });
 
-  it('Click to switch to another tab', () => {
+  it.skip('Click to switch to another tab', () => {
     cy.get('[data-test-subj="eventExplorer__addNewTab"]', {
       timeout: COMMAND_TIMEOUT_LONG,
     }).click();
@@ -441,7 +442,7 @@ describe('Live tail stop automatically', () => {
       .should('have.class', 'euiTab-isSelected');
   });
 
-  it('Close current selected tab', () => {
+  it.skip('Close current selected tab', () => {
     cy.get('[data-test-subj="eventExplorer__addNewTab"]', {
       timeout: COMMAND_TIMEOUT_LONG,
     }).click();

.github/workflows/dashboards-observability-test-and-build-workflow.yml

@@ -51,11 +51,15 @@ jobs:
         with:
           path: OpenSearch-Dashboards/plugins/dashboards-observability
 
-
       - name: Plugin Bootstrap
-        run: |
-          cd OpenSearch-Dashboards/plugins/dashboards-observability
-          yarn osd bootstrap
+        uses: nick-fields/retry@v2
+        with:
+          timeout_minutes: 20
+          max_attempts: 2
+          command: |
+            cd OpenSearch-Dashboards
+            yarn config set network-timeout 1000000 -g
+            yarn osd bootstrap
 
       - name: Test all dashboards-observability modules
         run: |

auto_sync_commit_metadata.json

@@ -1,4 +1,4 @@
 {
-    "last_github_commit": "cefd57fd94ddc916603128ae2cf7cf18c5962b38",
-    "last_gitfarm_commit": "3905d90c37023bc6d6e0a3508a46f8e99d353836"
+    "last_github_commit": "171acde2cd2468d619cdd9bd7051144197ebd89d",
+    "last_gitfarm_commit": "4b5220261e4f5ecf259856ff1963e8e29dd1644b"
 }

common/constants/data_connections.ts

@@ -7,8 +7,6 @@ import { DatasourceType } from '../../common/types/data_connections';
 
 export const OPENSEARCH_DOCUMENTATION_URL = 'https://opensearch.org/docs/latest/data-sources/index';
 
-export const OPENSEARCH_ACC_DOCUMENTATION_URL =
-  'https://opensearch.org/docs/latest/data-acceleration/index';
 export const QUERY_RESTRICTED = 'query-restricted';
 export const QUERY_ALL = 'query-all';
 

common/constants/explorer.ts

@@ -7,10 +7,13 @@ import { htmlIdGenerator } from '@elastic/eui';
 import { ThresholdUnitType } from '../../public/components/event_analytics/explorer/visualizations/config_panel/config_panes/config_controls/config_thresholds';
 import { VIS_CHART_TYPES } from './shared';
 
+// URLs
 export const EVENT_ANALYTICS_DOCUMENTATION_URL =
   'https://opensearch.org/docs/latest/observability-plugin/event-analytics/';
 export const OPEN_TELEMETRY_LOG_CORRELATION_LINK =
   'https://opentelemetry.io/docs/reference/specification/logs/overview/#log-correlation';
+export const LOG_EXPLORER_BASE_PATH = 'observability-logs#/explorer/';
+
 export const RAW_QUERY = 'rawQuery';
 export const FINAL_QUERY = 'finalQuery';
 export const SELECTED_DATE_RANGE = 'selectedDateRange';
@@ -50,6 +53,8 @@ export const DEFAULT_COLUMNS = ['', 'Time', '_source'];
 export const OTEL_TRACE_ID = 'traceId';
 export const JAEGER_TRACE_ID = 'traceID';
 export const DATE_PICKER_FORMAT = 'YYYY-MM-DD HH:mm:ss';
+export const DATE_DISPLAY_FORMAT = 'MMM D, YYYY @ HH:mm:ss.SSS';
+export const DEFAULT_DATETIME_STRING = 'now';
 export const TIME_INTERVAL_OPTIONS = [
   {
     text: 'Minute',
@@ -320,3 +325,23 @@ export const TYPE_TAB_MAPPING = {
   [SAVED_QUERY]: TAB_EVENT_ID,
   [SAVED_VISUALIZATION]: TAB_CHART_ID,
 };
+
+export const DEFAULT_EMPTY_EXPLORER_FIELDS = [
+  { name: 'timestamp', type: 'timestamp' },
+  { name: '_source', type: 'string' },
+];
+
+export const DEFAULT_TIMESTAMP_COLUMN = {
+  id: 'timestamp',
+  isSortable: true,
+  display: 'Time',
+  schema: 'datetime',
+  initialWidth: 200,
+};
+
+export const DEFAULT_SOURCE_COLUMN = {
+  id: '_source',
+  isSortable: false,
+  display: 'Source',
+  schema: '_source',
+};

common/constants/integrations.ts

@@ -5,3 +5,7 @@
 
 export const OPENSEARCH_DOCUMENTATION_URL = 'https://opensearch.org/docs/latest/integrations/index';
 export const ASSET_FILTER_OPTIONS = ['index-pattern', 'search', 'visualization', 'dashboard'];
+export const VALID_INDEX_NAME = /^[a-z\d\.][a-z\d\._\-\*]*$/;
+
+// Upstream doesn't export this, so we need to redeclare it for our use.
+export type Color = 'success' | 'primary' | 'warning' | 'danger' | undefined;

common/constants/shared.ts

@@ -3,7 +3,6 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 import CSS from 'csstype';
-import { IField } from '../../common/types/explorer';
 
 // Client route
 export const PPL_BASE = '/api/ppl';
@@ -14,20 +13,18 @@ export const DSL_CAT = '/cat.indices';
 export const DSL_MAPPING = '/indices.getFieldMapping';
 export const OBSERVABILITY_BASE = '/api/observability';
 export const INTEGRATIONS_BASE = '/api/integrations';
-export const JOBS_BASE = '/query/jobs';
 export const DATACONNECTIONS_BASE = '/api/dataconnections';
 export const EVENT_ANALYTICS = '/event_analytics';
 export const SAVED_OBJECTS = '/saved_objects';
 export const SAVED_QUERY = '/query';
 export const SAVED_VISUALIZATION = '/vis';
+export const CONSOLE_PROXY = '/api/console/proxy';
 
 // Server route
 export const PPL_ENDPOINT = '/_plugins/_ppl';
 export const SQL_ENDPOINT = '/_plugins/_sql';
 export const DSL_ENDPOINT = '/_plugins/_dsl';
 export const DATACONNECTIONS_ENDPOINT = '/_plugins/_query/_datasources';
-export const JOBS_ENDPOINT_BASE = '/_plugins/_async_query';
-export const JOB_RESULT_ENDPOINT = '/result';
 
 export const observabilityID = 'observability-logs';
 export const observabilityTitle = 'Observability';
@@ -61,6 +58,12 @@ export const observabilityIntegrationsID = 'integrations';
 export const observabilityIntegrationsTitle = 'Integrations';
 export const observabilityIntegrationsPluginOrder = 9020;
 
+export const observabilityDataConnectionsID = 'datasources';
+export const observabilityDataConnectionsTitle = 'Data sources';
+export const observabilityDataConnectionsPluginOrder = 9030;
+
+export const queryWorkbenchPluginID = 'opensearch-query-workbench';
+
 // Shared Constants
 export const SQL_DOCUMENTATION_URL = 'https://opensearch.org/docs/latest/search-plugins/sql/index/';
 export const PPL_DOCUMENTATION_URL =
@@ -79,18 +82,21 @@ export const PPL_NEWLINE_REGEX = /[\n\r]+/g;
 
 // Observability plugin URI
 const BASE_OBSERVABILITY_URI = '/_plugins/_observability';
-const BASE_INTEGRATIONS_URI = '/_plugins/_integrations'; // Used later in front-end for routing
+const BASE_DATACONNECTIONS_URI = '/_plugins/_query/_datasources';
 export const OPENSEARCH_PANELS_API = {
   OBJECT: `${BASE_OBSERVABILITY_URI}/object`,
 };
+export const OPENSEARCH_DATACONNECTIONS_API = {
+  DATACONNECTION: `${BASE_DATACONNECTIONS_URI}`,
+};
 
 // Saved Objects
 export const SAVED_OBJECT = '/object';
 
 // Color Constants
 export const PLOTLY_COLOR = [
   '#3CA1C7',
-  '#8C55A3',
+  '#54B399',
   '#DB748A',
   '#F2BE4B',
   '#68CCC2',

common/types/data_connections.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { EuiComboBoxOptionOption } from '@elastic/eui';
+
+export interface PermissionsConfigurationProps {
+  roles: Role[];
+  selectedRoles: Role[];
+  setSelectedRoles: React.Dispatch<React.SetStateAction<Role[]>>;
+  layout: 'horizontal' | 'vertical';
+}
+
+export type Role = EuiComboBoxOptionOption;
+
+export type DatasourceType = 'S3GLUE' | 'PROMETHEUS';

common/types/explorer.ts

@@ -92,9 +92,7 @@ export interface IExplorerFields {
 }
 
 export interface EmptyTabParams {
-  tabIds: string[] | undefined;
-  queries: any | undefined;
-  explorerData: any | undefined;
+  tabIds: string[];
 }
 
 export interface ILogExplorerProps {
@@ -383,3 +381,28 @@ export interface VisSpecificMetaData {
   x_coordinate: string;
   y_coordinate: string;
 }
+
+export type MOMENT_UNIT_OF_TIME =
+  | 'years'
+  | 'y'
+  | 'quarters'
+  | 'Q'
+  | 'months'
+  | 'M'
+  | 'weeks'
+  | 'w'
+  | 'days'
+  | 'd'
+  | 'hours'
+  | 'h'
+  | 'minutes'
+  | 'm'
+  | 'seconds'
+  | 's'
+  | 'milliseconds'
+  | 'ms';
+
+export interface GridSortingColumn {
+  id: string;
+  direction: 'asc' | 'desc';
+}

common/utils/index.ts

@@ -10,6 +10,6 @@ export {
   buildRawQuery,
   composeFinalQuery,
   removeBacktick,
-} from './query_utils';
+} from '../../public/components/common/query_utils';
 
 export * from './core_services';