Skip to content

Commit

Permalink
Update CI check for integ-test-with-security to run all integ tests w…
Browse files Browse the repository at this point in the history
…ith security (opensearch-project#1243)

* Run all integ tests with security

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Use adminClient when deleting ISM index in RestStopRollupActionIT

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Update RolloverActionIT

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Fix RestStopSnapshotManagementIT

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Fix RestChangePolicyActionIT

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Fix more tests

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Fix more tests

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Fix more tests

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Fix RestExplainTransformActionIT

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Fix more tests

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Make sure read from system index also uses adminClient

Signed-off-by: Craig Perkins <cwperx@amazon.com>

---------

Signed-off-by: Craig Perkins <cwperx@amazon.com>
(cherry picked from commit 395317d)
  • Loading branch information
cwperks committed Sep 11, 2024
1 parent 56ea466 commit 1626b26
Show file tree
Hide file tree
Showing 17 changed files with 96 additions and 59 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/security-test-workflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ jobs:
- name: Run integration tests
run: |
chown -R 1000:1000 `pwd`
su `id -un 1000` -c "./gradlew integTest -Dsecurity=true -Dhttps=true --tests '*SecurityBehaviorIT'"
su `id -un 1000` -c "./gradlew integTest -Dsecurity=true -Dhttps=true --tests '*IT'"
- name: Upload failed logs
uses: actions/upload-artifact@v2
if: failure()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -338,11 +338,12 @@ abstract class IndexStateManagementRestTestCase : IndexManagementRestTestCase()
{
"$key" : "$value"
}
""".trimIndent()
val res = client().makeRequest(
"PUT", "$index/_settings", emptyMap(),
StringEntity(body, APPLICATION_JSON),
)
""".trimIndent()
val res =
adminClient().makeRequest(
"PUT", "$index/_settings", emptyMap(),
StringEntity(body, ContentType.APPLICATION_JSON),
)
assertEquals("Update index setting failed", RestStatus.OK, res.restStatus())
}

Expand Down Expand Up @@ -465,13 +466,14 @@ abstract class IndexStateManagementRestTestCase : IndexManagementRestTestCase()
}

protected fun updateManagedIndexConfigPolicySeqNo(update: ManagedIndexConfig) {
val response = client().makeRequest(
"POST", "$INDEX_MANAGEMENT_INDEX/_update/${update.id}",
StringEntity(
"{\"doc\":{\"managed_index\":{\"policy_seq_no\":\"${update.policySeqNo}\"}}}",
APPLICATION_JSON,
),
)
val response =
adminClient().makeRequest(
"POST", "$INDEX_MANAGEMENT_INDEX/_update/${update.id}",
StringEntity(
"{\"doc\":{\"managed_index\":{\"policy_seq_no\":\"${update.policySeqNo}\"}}}",
APPLICATION_JSON,
),
)
assertEquals("Request failed", RestStatus.OK, response.restStatus())
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -770,21 +770,22 @@ class RolloverActionIT : IndexStateManagementRestTestCase() {
assertEquals("Index did not rollover.", AttemptRolloverStep.getSuccessMessage(firstIndex), info["message"])
}
// Manually produce transaction failure
val response = client().makeRequest(
"POST", "$INDEX_MANAGEMENT_INDEX/_update/${managedIndexConfig.id}%23metadata",
StringEntity(
"{\n" +
" \"script\": {\n" +
" \"source\": \"ctx._source.managed_index_metadata.step.step_status = params.step_status\",\n" +
" \"lang\": \"painless\",\n" +
" \"params\": {\n" +
" \"step_status\": \"starting\"\n" +
" }\n" +
" }\n" +
"}",
ContentType.APPLICATION_JSON,
),
)
val response =
adminClient().makeRequest(
"POST", "$INDEX_MANAGEMENT_INDEX/_update/${managedIndexConfig.id}%23metadata",
StringEntity(
"{\n" +
" \"script\": {\n" +
" \"source\": \"ctx._source.managed_index_metadata.step.step_status = params.step_status\",\n" +
" \"lang\": \"painless\",\n" +
" \"params\": {\n" +
" \"step_status\": \"starting\"\n" +
" }\n" +
" }\n" +
"}",
ContentType.APPLICATION_JSON,
),
)
assertEquals("Request failed", RestStatus.OK, response.restStatus())

// Execute again to see the transaction failure
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
package org.opensearch.indexmanagement.indexstatemanagement.resthandler

import org.junit.Before
import org.opensearch.client.Request
import org.opensearch.client.ResponseException
import org.opensearch.common.settings.Settings
import org.opensearch.core.rest.RestStatus
Expand Down Expand Up @@ -35,7 +36,7 @@ import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StateMetaDa
import org.opensearch.indexmanagement.waitFor
import org.opensearch.rest.RestRequest
import java.time.Instant
import java.util.Locale
import java.util.*

class RestChangePolicyActionIT : IndexStateManagementRestTestCase() {

Expand Down Expand Up @@ -87,7 +88,10 @@ class RestChangePolicyActionIT : IndexStateManagementRestTestCase() {
}

fun `test nonexistent ism config index`() {
if (indexExists(INDEX_MANAGEMENT_INDEX)) deleteIndex(INDEX_MANAGEMENT_INDEX)
if (indexExists(INDEX_MANAGEMENT_INDEX)) {
val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX")
adminClient().performRequest(deleteISMIndexRequest)
}
try {
val changePolicy = ChangePolicy("some_id", null, emptyList(), false)
client().makeRequest(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,13 @@

package org.opensearch.indexmanagement.rollup.resthandler

import org.opensearch.client.Request
import org.opensearch.client.ResponseException
import org.opensearch.common.settings.Settings
import org.opensearch.core.rest.RestStatus
import org.opensearch.indexmanagement.IndexManagementIndices
import org.opensearch.indexmanagement.IndexManagementPlugin
import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX
import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.ROLLUP_JOBS_BASE_URI
import org.opensearch.indexmanagement.common.model.dimension.DateHistogram
import org.opensearch.indexmanagement.indexstatemanagement.util.INDEX_HIDDEN
Expand All @@ -22,7 +24,7 @@ import org.opensearch.indexmanagement.waitFor
import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule
import java.time.Instant
import java.time.temporal.ChronoUnit
import java.util.Locale
import java.util.*

class RestStartRollupActionIT : RollupRestAPITestCase() {

Expand Down Expand Up @@ -199,7 +201,8 @@ class RestStartRollupActionIT : RollupRestAPITestCase() {

fun `test start rollup when multiple shards configured for IM config index`() {
// setup ism-config index with multiple primary shards
deleteIndex(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX)
val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX")
adminClient().performRequest(deleteISMIndexRequest)
val mapping = IndexManagementIndices.indexManagementMappings.trim().trimStart('{').trimEnd('}')
val settings = Settings.builder()
.put(INDEX_HIDDEN, true)
Expand Down Expand Up @@ -250,7 +253,7 @@ class RestStartRollupActionIT : RollupRestAPITestCase() {

// clearing the config index to prevent other tests using this multi shard index
Thread.sleep(2000L)
deleteIndex(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX)
adminClient().performRequest(deleteISMIndexRequest)
Thread.sleep(2000L)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,13 @@

package org.opensearch.indexmanagement.rollup.resthandler

import org.opensearch.client.Request
import org.opensearch.client.ResponseException
import org.opensearch.common.settings.Settings
import org.opensearch.core.rest.RestStatus
import org.opensearch.indexmanagement.IndexManagementIndices
import org.opensearch.indexmanagement.IndexManagementPlugin
import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX
import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.ROLLUP_JOBS_BASE_URI
import org.opensearch.indexmanagement.common.model.dimension.DateHistogram
import org.opensearch.indexmanagement.common.model.dimension.Terms
Expand Down Expand Up @@ -252,7 +254,8 @@ class RestStopRollupActionIT : RollupRestAPITestCase() {

fun `test stop rollup when multiple shards configured for IM config index`() {
// setup ism-config index with multiple primary shards
deleteIndex(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX)
val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX")
adminClient().performRequest(deleteISMIndexRequest)
val mapping = IndexManagementIndices.indexManagementMappings.trim().trimStart('{').trimEnd('}')
val settings = Settings.builder()
.put(INDEX_HIDDEN, true)
Expand Down Expand Up @@ -306,7 +309,7 @@ class RestStopRollupActionIT : RollupRestAPITestCase() {

// clearing the config index to prevent other tests using this multi shard index
Thread.sleep(2000L)
deleteIndex(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX)
adminClient().performRequest(deleteISMIndexRequest)
Thread.sleep(2000L)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -1394,7 +1394,7 @@ class RollupRunnerIT : RollupRestTestCase() {
// - Source index with pattern mapping to some closed indices

private fun deleteRollupMetadata(metadataId: String) {
val response = client().makeRequest("DELETE", "${IndexManagementPlugin.INDEX_MANAGEMENT_INDEX}/_doc/$metadataId")
val response = adminClient().makeRequest("DELETE", "${IndexManagementPlugin.INDEX_MANAGEMENT_INDEX}/_doc/$metadataId")
assertEquals("Unable to delete rollup metadata $metadataId", RestStatus.OK, response.restStatus())
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -138,13 +138,14 @@ abstract class SnapshotManagementRestTestCase : IndexManagementRestTestCase() {
val millis = Duration.of(intervalSchedule.interval.toLong(), intervalSchedule.unit).minusSeconds(2).toMillis()
val startTimeMillis = desiredStartTimeMillis ?: (now().toEpochMilli() - millis)
val waitForActiveShards = if (isMultiNode) "all" else "1"
val response = client().makeRequest(
"POST", "$INDEX_MANAGEMENT_INDEX/_update/${update.id}?wait_for_active_shards=$waitForActiveShards",
StringEntity(
"{\"doc\":{\"sm_policy\":{\"schedule\":{\"interval\":{\"start_time\":\"$startTimeMillis\"}}}}}",
APPLICATION_JSON,
),
)
val response =
adminClient().makeRequest(
"POST", "$INDEX_MANAGEMENT_INDEX/_update/${update.id}?wait_for_active_shards=$waitForActiveShards",
StringEntity(
"{\"doc\":{\"sm_policy\":{\"schedule\":{\"interval\":{\"start_time\":\"$startTimeMillis\"}}}}}",
APPLICATION_JSON,
),
)

assertEquals("Request failed", RestStatus.OK, response.restStatus())
}
Expand All @@ -169,10 +170,11 @@ abstract class SnapshotManagementRestTestCase : IndexManagementRestTestCase() {
val millis = Duration.of(intervalSchedule.interval.toLong(), intervalSchedule.unit).minusSeconds(2).toMillis()
val startTimeMillis = desiredStartTimeMillis ?: (now().toEpochMilli() - millis)
val waitForActiveShards = if (isMultiNode) "all" else "1"
val response = client().makeRequest(
"POST", "$INDEX_MANAGEMENT_INDEX/_update/${update.metadataID}?wait_for_active_shards=$waitForActiveShards",
StringEntity(
"""
val response =
adminClient().makeRequest(
"POST", "$INDEX_MANAGEMENT_INDEX/_update/${update.metadataID}?wait_for_active_shards=$waitForActiveShards",
StringEntity(
"""
{
"doc": {
"sm_metadata": {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

package org.opensearch.indexmanagement.snapshotmanagement.resthandler

import org.opensearch.client.Request
import org.opensearch.client.ResponseException
import org.opensearch.core.rest.RestStatus
import org.opensearch.indexmanagement.IndexManagementPlugin
Expand Down Expand Up @@ -39,7 +40,8 @@ class RestDeleteSnapshotManagementIT : SnapshotManagementRestTestCase() {

fun `test deleting a snapshot management policy that doesn't exist and config index doesnt exist`() {
try {
deleteIndex(INDEX_MANAGEMENT_INDEX)
val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX")
adminClient().performRequest(deleteISMIndexRequest)
client().makeRequest("DELETE", "${IndexManagementPlugin.SM_POLICIES_URI}/nonexistent_policy")
fail("expected 404 ResponseException")
} catch (e: ResponseException) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

package org.opensearch.indexmanagement.snapshotmanagement.resthandler

import org.opensearch.client.Request
import org.opensearch.client.ResponseException
import org.opensearch.common.xcontent.XContentType
import org.opensearch.core.rest.RestStatus
Expand Down Expand Up @@ -139,7 +140,8 @@ class RestExplainSnapshotManagementIT : SnapshotManagementRestTestCase() {

fun `test explain sm policy when config index doesn't exist`() {
try {
deleteIndex(INDEX_MANAGEMENT_INDEX)
val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX")
adminClient().performRequest(deleteISMIndexRequest)
explainSMPolicy(randomAlphaOfLength(10))
fail("expected response exception")
} catch (e: ResponseException) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,11 @@ package org.opensearch.indexmanagement.snapshotmanagement.resthandler

import org.apache.http.HttpHeaders
import org.apache.http.message.BasicHeader
import org.opensearch.client.Request
import org.opensearch.client.ResponseException
import org.opensearch.core.rest.RestStatus
import org.opensearch.indexmanagement.IndexManagementPlugin
import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX
import org.opensearch.indexmanagement.makeRequest
import org.opensearch.indexmanagement.opensearchapi.convertToMap
import org.opensearch.indexmanagement.snapshotmanagement.SnapshotManagementRestTestCase
Expand Down Expand Up @@ -48,7 +50,8 @@ class RestGetSnapshotManagementIT : SnapshotManagementRestTestCase() {
@Throws(Exception::class)
fun `test getting a snapshot management policy that doesn't exist and config index doesnt exist`() {
try {
deleteIndex(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX)
val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX")
adminClient().performRequest(deleteISMIndexRequest)
getSMPolicy(randomAlphaOfLength(20))
fail("expected response exception")
} catch (e: ResponseException) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

package org.opensearch.indexmanagement.snapshotmanagement.resthandler

import org.opensearch.client.Request
import org.opensearch.client.ResponseException
import org.opensearch.common.xcontent.XContentType
import org.opensearch.core.rest.RestStatus
Expand Down Expand Up @@ -129,7 +130,8 @@ class RestIndexSnapshotManagementIT : SnapshotManagementRestTestCase() {
@Throws(Exception::class)
@Suppress("UNCHECKED_CAST")
fun `test mappings after sm policy creation`() {
deleteIndex(INDEX_MANAGEMENT_INDEX)
val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX")
adminClient().performRequest(deleteISMIndexRequest)
createSMPolicy(randomSMPolicy())

val response = client().makeRequest("GET", "/$INDEX_MANAGEMENT_INDEX/_mapping")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,11 @@

package org.opensearch.indexmanagement.snapshotmanagement.resthandler

import org.opensearch.client.Request
import org.opensearch.client.ResponseException
import org.opensearch.core.rest.RestStatus
import org.opensearch.indexmanagement.IndexManagementPlugin
import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX
import org.opensearch.indexmanagement.makeRequest
import org.opensearch.indexmanagement.snapshotmanagement.SnapshotManagementRestTestCase
import org.opensearch.indexmanagement.snapshotmanagement.randomSMPolicy
Expand Down Expand Up @@ -60,7 +62,8 @@ class RestStartSnapshotManagementIT : SnapshotManagementRestTestCase() {

fun `test starting a snapshot management policy with no config index fails`() {
try {
deleteIndex(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX)
val request = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX")
adminClient().performRequest(request)
client().makeRequest("POST", "${IndexManagementPlugin.SM_POLICIES_URI}/nonexistent_foo/_start")
fail("expected response exception")
} catch (e: ResponseException) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,11 @@

package org.opensearch.indexmanagement.snapshotmanagement.resthandler

import org.opensearch.client.Request
import org.opensearch.client.ResponseException
import org.opensearch.core.rest.RestStatus
import org.opensearch.indexmanagement.IndexManagementPlugin
import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX
import org.opensearch.indexmanagement.makeRequest
import org.opensearch.indexmanagement.snapshotmanagement.SnapshotManagementRestTestCase
import org.opensearch.indexmanagement.snapshotmanagement.randomSMPolicy
Expand Down Expand Up @@ -60,7 +62,8 @@ class RestStopSnapshotManagementIT : SnapshotManagementRestTestCase() {

fun `test stopping a snapshot management policy with no config index fails`() {
try {
deleteIndex(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX)
val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX")
adminClient().performRequest(deleteISMIndexRequest)
client().makeRequest("POST", "${IndexManagementPlugin.SM_POLICIES_URI}/nonexistent_foo/_stop")
fail("expected response exception")
} catch (e: ResponseException) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -116,9 +116,10 @@ abstract class TransformRestTestCase : IndexManagementRestTestCase() {
}

protected fun getTransformMetadata(metadataId: String): TransformMetadata {
val response = client().makeRequest(
"GET", "$INDEX_MANAGEMENT_INDEX/_doc/$metadataId", null, BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json"),
)
val response =
adminClient().makeRequest(
"GET", "$INDEX_MANAGEMENT_INDEX/_doc/$metadataId", null, BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json"),
)
assertEquals("Unable to get transform metadata $metadataId", RestStatus.OK, response.restStatus())

val parser = createParser(XContentType.JSON.xContent(), response.entity.content)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

package org.opensearch.indexmanagement.transform.resthandler

import org.opensearch.client.Request
import org.opensearch.client.ResponseException
import org.opensearch.core.rest.RestStatus
import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX
Expand Down Expand Up @@ -75,7 +76,10 @@ class RestDeleteTransformActionIT : TransformRestTestCase() {
@Throws(Exception::class)
fun `test deleting a transform that doesn't exist and config index doesn't exist`() {
try {
if (indexExists(INDEX_MANAGEMENT_INDEX)) deleteIndex(INDEX_MANAGEMENT_INDEX)
if (indexExists(INDEX_MANAGEMENT_INDEX)) {
val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX")
adminClient().performRequest(deleteISMIndexRequest)
}
val res = client().makeRequest("DELETE", "$TRANSFORM_BASE_URI/foobarbaz")
fail("expected 404 ResponseException: ${res.asMap()}")
} catch (e: ResponseException) {
Expand Down
Loading

0 comments on commit 1626b26

Please sign in to comment.