diff --git a/packages/api/api.go b/packages/api/api.go index 325c1c2e..af9d5304 100644 --- a/packages/api/api.go +++ b/packages/api/api.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" "net/url" + "strconv" "strings" "github.com/Infisical/infisical-merge/packages/config" @@ -598,36 +599,31 @@ func CallMachineIdentityRefreshAccessToken(httpClient *resty.Client, request Uni return universalAuthRefreshResponse, nil } -func CallGetRawSecretsV3(httpClient *resty.Client, request GetRawSecretsV3Request) (GetRawSecretsV3Response, error) { - var getRawSecretsV3Response GetRawSecretsV3Response +func CallGetSecretsV4(httpClient *resty.Client, request GetSecretsV4Request) (GetSecretsV4Response, error) { + var getRawSecretsV4Response GetSecretsV4Response req := httpClient. R(). - SetResult(&getRawSecretsV3Response). + SetResult(&getRawSecretsV4Response). SetHeader("User-Agent", USER_AGENT). SetBody(request). - SetQueryParam("workspaceId", request.WorkspaceId). + SetQueryParam("projectId", request.WorkspaceId). SetQueryParam("environment", request.Environment). - SetQueryParam("secretPath", request.SecretPath) + SetQueryParam("secretPath", request.SecretPath). + // v4 defaults these to true, so they must always be sent explicitly to honor a false flag + SetQueryParam("includeImports", strconv.FormatBool(request.IncludeImport)). + SetQueryParam("recursive", strconv.FormatBool(request.Recursive)). + SetQueryParam("expandSecretReferences", strconv.FormatBool(request.ExpandSecretReferences)). + // v4 resolves personal overrides server-side (Priority when true, NeverInclude when false) + SetQueryParam("includePersonalOverrides", strconv.FormatBool(request.IncludePersonalOverrides)) if request.TagSlugs != "" { req.SetQueryParam("tagSlugs", request.TagSlugs) } - if request.IncludeImport { - req.SetQueryParam("include_imports", "true") - } - if request.Recursive { - req.SetQueryParam("recursive", "true") - } - - if request.ExpandSecretReferences { - req.SetQueryParam("expandSecretReferences", "true") - } - - response, err := req.Get(fmt.Sprintf("%v/v3/secrets/raw", config.INFISICAL_URL)) + response, err := req.Get(fmt.Sprintf("%v/v4/secrets", config.INFISICAL_URL)) if err != nil { - return GetRawSecretsV3Response{}, NewGenericRequestError(operationCallGetRawSecretsV3, err) + return GetSecretsV4Response{}, NewGenericRequestError(operationCallGetRawSecretsV3, err) } if response.IsError() && @@ -635,44 +631,44 @@ func CallGetRawSecretsV3(httpClient *resty.Client, request GetRawSecretsV3Reques strings.Contains(strings.ToLower(response.String()), "failed to find bot key") || strings.Contains(strings.ToLower(response.String()), "bot is not active")) { additionalContext := fmt.Sprintf(`Project with id %s is incompatible with your current CLI version. Upgrade your project by visiting the project settings page. If you're self-hosting and project upgrade option isn't yet available, contact your administrator to upgrade your Infisical instance to the latest release.`, request.WorkspaceId) - return GetRawSecretsV3Response{}, NewAPIErrorWithResponse(operationCallGetRawSecretsV3, response, &additionalContext) + return GetSecretsV4Response{}, NewAPIErrorWithResponse(operationCallGetRawSecretsV3, response, &additionalContext) } if response.IsError() { - return GetRawSecretsV3Response{}, NewAPIErrorWithResponse(operationCallGetRawSecretsV3, response, nil) + return GetSecretsV4Response{}, NewAPIErrorWithResponse(operationCallGetRawSecretsV3, response, nil) } - getRawSecretsV3Response.ETag = response.Header().Get(("etag")) + getRawSecretsV4Response.ETag = response.Header().Get(("etag")) - return getRawSecretsV3Response, nil + return getRawSecretsV4Response, nil } -func CallFetchSingleSecretByName(httpClient *resty.Client, request GetRawSecretV3ByNameRequest) (GetRawSecretV3ByNameResponse, error) { - var getRawSecretV3ByNameResponse GetRawSecretV3ByNameResponse +func CallFetchSingleSecretByName(httpClient *resty.Client, request GetSecretV4ByNameRequest) (GetSecretV4ByNameResponse, error) { + var getSecretV4ByNameResponse GetSecretV4ByNameResponse response, err := httpClient. R(). SetHeader("User-Agent", USER_AGENT). - SetResult(&getRawSecretV3ByNameResponse). + SetResult(&getSecretV4ByNameResponse). SetBody(request). SetQueryParam("expandSecretReferences", "true"). - SetQueryParam("include_imports", "true"). + SetQueryParam("includeImports", "true"). SetQueryParam("environment", request.Environment). SetQueryParam("secretPath", request.SecretPath). - SetQueryParam("workspaceId", request.WorkspaceID). + SetQueryParam("projectId", request.WorkspaceID). SetQueryParam("type", "shared"). - Get(fmt.Sprintf("%v/v3/secrets/raw/%s", config.INFISICAL_URL, request.SecretName)) + Get(fmt.Sprintf("%v/v4/secrets/%s", config.INFISICAL_URL, request.SecretName)) if err != nil { - return GetRawSecretV3ByNameResponse{}, NewGenericRequestError(operationCallFetchSingleSecretByName, err) + return GetSecretV4ByNameResponse{}, NewGenericRequestError(operationCallFetchSingleSecretByName, err) } if response.IsError() { - return GetRawSecretV3ByNameResponse{}, NewAPIErrorWithResponse(operationCallFetchSingleSecretByName, response, nil) + return GetSecretV4ByNameResponse{}, NewAPIErrorWithResponse(operationCallFetchSingleSecretByName, response, nil) } - getRawSecretV3ByNameResponse.ETag = response.Header().Get(("etag")) + getSecretV4ByNameResponse.ETag = response.Header().Get(("etag")) - return getRawSecretV3ByNameResponse, nil + return getSecretV4ByNameResponse, nil } func CallCreateDynamicSecretLeaseV1(httpClient *resty.Client, request CreateDynamicSecretLeaseV1Request) (CreateDynamicSecretLeaseV1Response, error) { diff --git a/packages/api/model.go b/packages/api/model.go index f7d116c0..7e4cb85b 100644 --- a/packages/api/model.go +++ b/packages/api/model.go @@ -663,17 +663,18 @@ type GetLoginV3Response struct { AccessToken string `json:"accessToken"` } -type GetRawSecretsV3Request struct { - Environment string `json:"environment"` - WorkspaceId string `json:"workspaceId"` - SecretPath string `json:"secretPath"` - IncludeImport bool `json:"include_imports"` - Recursive bool `json:"recursive"` - TagSlugs string `json:"tagSlugs,omitempty"` - ExpandSecretReferences bool `json:"expandSecretReferences,omitempty"` -} - -type GetRawSecretsV3Response struct { +type GetSecretsV4Request struct { + Environment string `json:"environment"` + WorkspaceId string `json:"projectId"` + SecretPath string `json:"secretPath"` + IncludeImport bool `json:"includeImports"` + Recursive bool `json:"recursive"` + TagSlugs string `json:"tagSlugs,omitempty"` + ExpandSecretReferences bool `json:"expandSecretReferences"` + IncludePersonalOverrides bool `json:"includePersonalOverrides"` +} + +type GetSecretsV4Response struct { Secrets []struct { ID string `json:"_id"` Version int `json:"version"` @@ -691,7 +692,7 @@ type GetRawSecretsV3Response struct { ETag string } -type GetRawSecretV3ByNameRequest struct { +type GetSecretV4ByNameRequest struct { SecretName string `json:"secretName"` WorkspaceID string `json:"workspaceId"` Type string `json:"type,omitempty"` @@ -699,7 +700,7 @@ type GetRawSecretV3ByNameRequest struct { SecretPath string `json:"secretPath,omitempty"` } -type GetRawSecretV3ByNameResponse struct { +type GetSecretV4ByNameResponse struct { Secret struct { ID string `json:"_id"` Version int `json:"version"` diff --git a/packages/cmd/agent.go b/packages/cmd/agent.go index 9d52195a..93ce80e5 100644 --- a/packages/cmd/agent.go +++ b/packages/cmd/agent.go @@ -908,7 +908,7 @@ func secretTemplateFunction(accessToken string, currentEtag *string) func(string parsedArguments.SetDefaults() - res, err := util.GetPlainTextSecretsV3(accessToken, projectID, envSlug, secretPath, true, parsedArguments.IsRecursive, "", *parsedArguments.ShouldExpandSecretReferences) + res, err := util.GetPlainTextSecretsV4(accessToken, projectID, envSlug, secretPath, true, parsedArguments.IsRecursive, "", *parsedArguments.ShouldExpandSecretReferences, false) if err != nil { return nil, err } diff --git a/packages/cmd/export.go b/packages/cmd/export.go index a75707e6..776074f2 100644 --- a/packages/cmd/export.go +++ b/packages/cmd/export.go @@ -94,12 +94,13 @@ var exportCmd = &cobra.Command{ } request := models.GetAllSecretsParameters{ - Environment: environmentName, - TagSlugs: tagSlugs, - WorkspaceId: projectId, - SecretsPath: secretsPath, - IncludeImport: includeImports, - ExpandSecretReferences: shouldExpandSecrets, + Environment: environmentName, + TagSlugs: tagSlugs, + WorkspaceId: projectId, + SecretsPath: secretsPath, + IncludeImport: includeImports, + ExpandSecretReferences: shouldExpandSecrets, + IncludePersonalOverrides: secretOverriding, } if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER { @@ -137,12 +138,6 @@ var exportCmd = &cobra.Command{ util.HandleError(err, "Unable to fetch secrets") } - if secretOverriding { - secrets = util.OverrideSecrets(secrets, util.SECRET_TYPE_PERSONAL) - } else { - secrets = util.OverrideSecrets(secrets, util.SECRET_TYPE_SHARED) - } - var output string secrets = util.FilterSecretsByTag(secrets, tagSlugs) secrets = util.SortSecretsByKeys(secrets) diff --git a/packages/cmd/run.go b/packages/cmd/run.go index 1de48cd2..db98bf4d 100644 --- a/packages/cmd/run.go +++ b/packages/cmd/run.go @@ -444,13 +444,14 @@ func fetchSecrets(request models.GetMultiPathSecretsParameters, projectConfigDir for _, path := range request.SecretsPaths { params := models.GetAllSecretsParameters{ - Environment: request.Environment, - WorkspaceId: request.WorkspaceId, - TagSlugs: request.TagSlugs, - SecretsPath: path, - IncludeImport: request.IncludeImport, - Recursive: request.Recursive, - ExpandSecretReferences: request.ExpandSecretReferences, + Environment: request.Environment, + WorkspaceId: request.WorkspaceId, + TagSlugs: request.TagSlugs, + SecretsPath: path, + IncludeImport: request.IncludeImport, + Recursive: request.Recursive, + ExpandSecretReferences: request.ExpandSecretReferences, + IncludePersonalOverrides: secretOverriding, } if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER { @@ -467,12 +468,6 @@ func fetchSecrets(request models.GetMultiPathSecretsParameters, projectConfigDir allSecrets = append(allSecrets, secrets...) } - if secretOverriding { - allSecrets = util.OverrideSecrets(allSecrets, util.SECRET_TYPE_PERSONAL) - } else { - allSecrets = util.OverrideSecrets(allSecrets, util.SECRET_TYPE_SHARED) - } - return allSecrets, nil } diff --git a/packages/cmd/secrets.go b/packages/cmd/secrets.go index 55123ba3..def0bd24 100644 --- a/packages/cmd/secrets.go +++ b/packages/cmd/secrets.go @@ -84,13 +84,14 @@ var secretsCmd = &cobra.Command{ } request := models.GetAllSecretsParameters{ - Environment: environmentName, - WorkspaceId: projectId, - TagSlugs: tagSlugs, - SecretsPath: secretsPath, - IncludeImport: includeImports, - Recursive: recursive, - ExpandSecretReferences: shouldExpandSecrets, + Environment: environmentName, + WorkspaceId: projectId, + TagSlugs: tagSlugs, + SecretsPath: secretsPath, + IncludeImport: includeImports, + Recursive: recursive, + ExpandSecretReferences: shouldExpandSecrets, + IncludePersonalOverrides: secretOverriding, } if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER { @@ -104,12 +105,6 @@ var secretsCmd = &cobra.Command{ util.HandleError(err) } - if secretOverriding { - secrets = util.OverrideSecrets(secrets, util.SECRET_TYPE_PERSONAL) - } else { - secrets = util.OverrideSecrets(secrets, util.SECRET_TYPE_SHARED) - } - // Sort the secrets by key so we can create a consistent output secrets = util.SortSecretsByKeys(secrets) @@ -505,13 +500,14 @@ func getSecretsByNames(cmd *cobra.Command, args []string) { } request := models.GetAllSecretsParameters{ - Environment: environmentName, - WorkspaceId: projectId, - TagSlugs: tagSlugs, - SecretsPath: secretsPath, - IncludeImport: includeImports, - Recursive: recursive, - ExpandSecretReferences: shouldExpand, + Environment: environmentName, + WorkspaceId: projectId, + TagSlugs: tagSlugs, + SecretsPath: secretsPath, + IncludeImport: includeImports, + Recursive: recursive, + ExpandSecretReferences: shouldExpand, + IncludePersonalOverrides: secretOverriding, } if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER { @@ -525,12 +521,6 @@ func getSecretsByNames(cmd *cobra.Command, args []string) { util.HandleError(err, "To fetch all secrets") } - if secretOverriding { - secrets = util.OverrideSecrets(secrets, util.SECRET_TYPE_PERSONAL) - } else { - secrets = util.OverrideSecrets(secrets, util.SECRET_TYPE_SHARED) - } - requestedSecrets := []models.SingleEnvironmentVariable{} secretsMap := getSecretsByKeys(secrets) diff --git a/packages/models/cli.go b/packages/models/cli.go index 16aed48a..a739d63c 100644 --- a/packages/models/cli.go +++ b/packages/models/cli.go @@ -142,6 +142,7 @@ type GetAllSecretsParameters struct { IncludeImport bool Recursive bool ExpandSecretReferences bool + IncludePersonalOverrides bool } type InjectableEnvironmentResult struct { diff --git a/packages/util/secrets.go b/packages/util/secrets.go index 5b1a884e..5ad4338c 100644 --- a/packages/util/secrets.go +++ b/packages/util/secrets.go @@ -20,7 +20,7 @@ import ( "gopkg.in/yaml.v3" ) -func GetPlainTextSecretsViaServiceToken(fullServiceToken string, environment string, secretPath string, includeImports bool, recursive bool, tagSlugs string, expandSecretReferences bool) ([]models.SingleEnvironmentVariable, error) { +func GetPlainTextSecretsViaServiceToken(fullServiceToken string, environment string, secretPath string, includeImports bool, recursive bool, tagSlugs string, expandSecretReferences bool, includePersonalOverrides bool) ([]models.SingleEnvironmentVariable, error) { serviceTokenParts := strings.SplitN(fullServiceToken, ".", 4) if len(serviceTokenParts) < 4 { return nil, fmt.Errorf("invalid service token entered. Please double check your service token and try again") @@ -50,14 +50,15 @@ func GetPlainTextSecretsViaServiceToken(fullServiceToken string, environment str } } - rawSecrets, err := api.CallGetRawSecretsV3(httpClient, api.GetRawSecretsV3Request{ - WorkspaceId: serviceTokenDetails.Workspace, - Environment: environment, - SecretPath: secretPath, - IncludeImport: includeImports, - Recursive: recursive, - TagSlugs: tagSlugs, - ExpandSecretReferences: expandSecretReferences, + rawSecrets, err := api.CallGetSecretsV4(httpClient, api.GetSecretsV4Request{ + WorkspaceId: serviceTokenDetails.Workspace, + Environment: environment, + SecretPath: secretPath, + IncludeImport: includeImports, + Recursive: recursive, + TagSlugs: tagSlugs, + ExpandSecretReferences: expandSecretReferences, + IncludePersonalOverrides: includePersonalOverrides, }) if err != nil { @@ -81,7 +82,7 @@ func GetPlainTextSecretsViaServiceToken(fullServiceToken string, environment str } -func GetPlainTextSecretsV3(accessToken string, workspaceId string, environmentName string, secretsPath string, includeImports bool, recursive bool, tagSlugs string, expandSecretReferences bool) (models.PlaintextSecretResult, error) { +func GetPlainTextSecretsV4(accessToken string, workspaceId string, environmentName string, secretsPath string, includeImports bool, recursive bool, tagSlugs string, expandSecretReferences bool, includePersonalOverrides bool) (models.PlaintextSecretResult, error) { httpClient, err := GetRestyClientWithCustomHeaders() if err != nil { return models.PlaintextSecretResult{}, err @@ -90,20 +91,21 @@ func GetPlainTextSecretsV3(accessToken string, workspaceId string, environmentNa httpClient.SetAuthToken(accessToken). SetHeader("Accept", "application/json") - getSecretsRequest := api.GetRawSecretsV3Request{ - WorkspaceId: workspaceId, - Environment: environmentName, - IncludeImport: includeImports, - Recursive: recursive, - TagSlugs: tagSlugs, - ExpandSecretReferences: expandSecretReferences, + getSecretsRequest := api.GetSecretsV4Request{ + WorkspaceId: workspaceId, + Environment: environmentName, + IncludeImport: includeImports, + Recursive: recursive, + TagSlugs: tagSlugs, + ExpandSecretReferences: expandSecretReferences, + IncludePersonalOverrides: includePersonalOverrides, } if secretsPath != "" { getSecretsRequest.SecretPath = secretsPath } - rawSecrets, err := api.CallGetRawSecretsV3(httpClient, getSecretsRequest) + rawSecrets, err := api.CallGetSecretsV4(httpClient, getSecretsRequest) if err != nil { return models.PlaintextSecretResult{}, err @@ -137,7 +139,7 @@ func GetSinglePlainTextSecretByNameV3(accessToken string, workspaceId string, en httpClient.SetAuthToken(accessToken). SetHeader("Accept", "application/json") - getSecretsRequest := api.GetRawSecretV3ByNameRequest{ + getSecretsRequest := api.GetSecretV4ByNameRequest{ WorkspaceID: workspaceId, Environment: environmentName, SecretName: secretName, @@ -331,8 +333,8 @@ func GetAllEnvironmentVariables(params models.GetAllSecretsParameters, projectCo params.WorkspaceId = infisicalDotJson.WorkspaceId } - res, err := GetPlainTextSecretsV3(loggedInUserDetails.UserCredentials.JTWToken, params.WorkspaceId, - params.Environment, params.SecretsPath, params.IncludeImport, params.Recursive, params.TagSlugs, true) + res, err := GetPlainTextSecretsV4(loggedInUserDetails.UserCredentials.JTWToken, params.WorkspaceId, + params.Environment, params.SecretsPath, params.IncludeImport, params.Recursive, params.TagSlugs, true, params.IncludePersonalOverrides) log.Debug().Msgf("GetAllEnvironmentVariables: Trying to fetch secrets JTW token [err=%s]", err) if err == nil { @@ -361,7 +363,7 @@ func GetAllEnvironmentVariables(params models.GetAllSecretsParameters, projectCo } else { if params.InfisicalToken != "" { log.Debug().Msg("Trying to fetch secrets using service token") - secretsToReturn, errorToReturn = GetPlainTextSecretsViaServiceToken(params.InfisicalToken, params.Environment, params.SecretsPath, params.IncludeImport, params.Recursive, params.TagSlugs, params.ExpandSecretReferences) + secretsToReturn, errorToReturn = GetPlainTextSecretsViaServiceToken(params.InfisicalToken, params.Environment, params.SecretsPath, params.IncludeImport, params.Recursive, params.TagSlugs, params.ExpandSecretReferences, params.IncludePersonalOverrides) } else if params.UniversalAuthAccessToken != "" { if params.WorkspaceId == "" { @@ -369,7 +371,7 @@ func GetAllEnvironmentVariables(params models.GetAllSecretsParameters, projectCo } log.Debug().Msg("Trying to fetch secrets using universal auth") - res, err := GetPlainTextSecretsV3(params.UniversalAuthAccessToken, params.WorkspaceId, params.Environment, params.SecretsPath, params.IncludeImport, params.Recursive, params.TagSlugs, params.ExpandSecretReferences) + res, err := GetPlainTextSecretsV4(params.UniversalAuthAccessToken, params.WorkspaceId, params.Environment, params.SecretsPath, params.IncludeImport, params.Recursive, params.TagSlugs, params.ExpandSecretReferences, params.IncludePersonalOverrides) errorToReturn = err secretsToReturn = res.Secrets @@ -379,49 +381,6 @@ func GetAllEnvironmentVariables(params models.GetAllSecretsParameters, projectCo return secretsToReturn, errorToReturn } -func OverrideSecrets(secrets []models.SingleEnvironmentVariable, secretType string) []models.SingleEnvironmentVariable { - personalSecrets := make(map[string]models.SingleEnvironmentVariable) - sharedSecrets := make(map[string]models.SingleEnvironmentVariable) - secretsToReturn := []models.SingleEnvironmentVariable{} - secretsToReturnMap := make(map[string]models.SingleEnvironmentVariable) - - for _, secret := range secrets { - if secret.Type == PERSONAL_SECRET_TYPE_NAME { - personalSecrets[secret.Key] = secret - } - if secret.Type == SHARED_SECRET_TYPE_NAME { - sharedSecrets[secret.Key] = secret - } - } - - if secretType == PERSONAL_SECRET_TYPE_NAME { - for _, secret := range secrets { - if personalSecret, exists := personalSecrets[secret.Key]; exists { - secretsToReturnMap[secret.Key] = personalSecret - } else { - if _, exists = secretsToReturnMap[secret.Key]; !exists { - secretsToReturnMap[secret.Key] = secret - } - } - } - } else if secretType == SHARED_SECRET_TYPE_NAME { - for _, secret := range secrets { - if sharedSecret, exists := sharedSecrets[secret.Key]; exists { - secretsToReturnMap[secret.Key] = sharedSecret - } else { - if _, exists := secretsToReturnMap[secret.Key]; !exists { - secretsToReturnMap[secret.Key] = secret - } - } - } - } - - for _, secret := range secretsToReturnMap { - secretsToReturn = append(secretsToReturn, secret) - } - return secretsToReturn -} - func GetBackupEncryptionKey() ([]byte, error) { encryptionKey, err := GetValueInKeyring(INFISICAL_BACKUP_SECRET_ENCRYPTION_KEY) if err != nil { @@ -659,7 +618,7 @@ func SetRawSecrets(secretArgs []string, secretType string, environmentName strin return nil, fmt.Errorf("unable to process set secret operations, token details are missing") } - getAllEnvironmentVariablesRequest := models.GetAllSecretsParameters{Environment: environmentName, SecretsPath: secretsPath, WorkspaceId: projectId} + getAllEnvironmentVariablesRequest := models.GetAllSecretsParameters{Environment: environmentName, SecretsPath: secretsPath, WorkspaceId: projectId, IncludePersonalOverrides: secretType == "personal"} if tokenDetails.Type == UNIVERSAL_AUTH_TOKEN_IDENTIFIER { getAllEnvironmentVariablesRequest.UniversalAuthAccessToken = tokenDetails.Token } diff --git a/test/.snapshots/test-TestUniversalAuth_SecretsGetWrongEnvironment b/test/.snapshots/test-TestUniversalAuth_SecretsGetWrongEnvironment index ca675a22..1f4c5fba 100644 --- a/test/.snapshots/test-TestUniversalAuth_SecretsGetWrongEnvironment +++ b/test/.snapshots/test-TestUniversalAuth_SecretsGetWrongEnvironment @@ -1,4 +1,4 @@ -Request: GET https://app.infisical.com/api/v3/secrets/raw?environment=invalid-env&expandSecretReferences=true&include_imports=true&recursive=true&secretPath=%2F&workspaceId=bef697d4-849b-4a75-b284-0922f87f8ba2 +Request: GET https://app.infisical.com/api/v4/secrets?environment=invalid-env&expandSecretReferences=true&includeImports=true&includePersonalOverrides=true&projectId=bef697d4-849b-4a75-b284-0922f87f8ba2&recursive=true&secretPath=%2F Instance: https://app.infisical.com Request ID: Response Code: 404 Not Found