credential:
project_id: project-id-1234567
private_key_id: "d0121d0000882411234e11166c6aaa23ed5d74e0"
private_key: "-----BEGIN PRIVATE KEY-----\nMIIyourkey\n-----END PRIVATE KEY-
+
+
+# Case 4: NOT MATCH
+-BEGIN PGP PRIVATE KEY BLOCK-----\r\n"),p.push(f(u)),p.push(i.default.encode(t)),p.push("\r\n=",d(h),"\r\n"),p.push("-----END PGP PRIVATE KEY BLOCK-
\ No newline at end of file
diff --git a/tests/fixtures/cases/secret_in_querystring.py b/tests/fixtures/cases/secret_in_querystring.py
new file mode 100644
index 0000000..84013a9
--- /dev/null
+++ b/tests/fixtures/cases/secret_in_querystring.py
@@ -0,0 +1,7 @@
+ip_sources = ["Auto select", "ipinfo.io", "IP-API", "ipstack"]
+ip_urls = [
+ "",
+ "http://ipinfo.io/json",
+ "http://ip-api.com/json",
+ "http://api.ipstack.com/check?access_key=8g0gi323hg3036l3h4j0555jk7527970",
+]
diff --git a/tests/fixtures/cases/secret_in_yml_string.yml b/tests/fixtures/cases/secret_in_yml_string.yml
new file mode 100644
index 0000000..fece9a4
--- /dev/null
+++ b/tests/fixtures/cases/secret_in_yml_string.yml
@@ -0,0 +1,54 @@
+---
+http_interactions:
+- request:
+ method: post
+ uri: https://circleci.com/api/v1.1/project/mtchavez/circleci/ssh-key?circle-token=
+ body:
+ encoding: UTF-8
+ string: '{"hostname":"hostname","private_key":"-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAwwuwmtxV5vZwAuZKSEPrgddDb7Qv8eeX0PnCtS\nt428lZGSF0cNSmQIVIxBgi7twmLKpFgqS1LAAAb9+lXW+yh/f5Fl4VBirEvC3olS\n6yUAxWhDQ27waOK7ptTkYYGw+9g7QWSjjeK+vwIDAQABAoIBAG+oa4vcA1lbTzh5\kH1\nj2DjPlOi+Tno5OupI7xPY4HqHokToGkXxmKScxSnIefibATTzkRgVvIsyWjRxYbI\nZOuonGEkEUDmmvhhkNbuTqBd0XHcZxG8EJaeGsTLiJG4fG9x3ib/o4w7Lrfh2usb\nCjQt52L5Hg3JHXNokpwFOvhTN4uXUmjZKTQGmJceRz0Rqcemwn0ulcTYrQwQGyw/\nzncwkaNbiWRLJZJtRbMoTd3Mhr8ZvKp91vZGGLhn/BE2s7D5d/OXgz5N11T8iyN8\nSCW5YaECgYEA+Rlrlhui+tan9vo39VXh7wqJujrxlNTLP70ImpAfIoVVhAuPg+pL\naEPs/HIzSJOYppyI3fv6Bz8lKK46gO5VouokxmNQFNz2bxHn2FaXt37k/ye4tbqQ\nCuou+UvAlR/r4OT0Yj9wla0P5wo7IWRrdwWiAhj1DSkvkyGL3VLxF8kCgYEAyHLt\no43wRg50l63gectGCSRfyXuml3o38BLwQhToqbqPQFVncur+lNar9Xh5FRO9T+am\nd8ET+GLdCUXxfpu6eytEUgG3yndsHgXJz5Ta2hQdQT4lkkOlirO4ozYrBIilSrZh\nr8CQP2GKrDZ0QZkZRsBYKUZ0OnNPQXyboCfa9kcCgYB79mCJ9ProZYZ07BSI7NJg\nyRe9K7QpYrQ65fGwKWS1IzFpYu9qsGASZSs8fgBzb7AZyfB6t/i3Pn9ZfUrz+qd3\nSZo/eBDUMRoaMAj2qjEaSfXf3H6ZQVyJcf0qZr5R9+7EnmvXsMZwVg2B5p+CgJzS\nQGVdMdpRUFuylpEp9SqxsQKBgDkuB7obOEpTv1.1CxahJ0ORNMjAKGwlv9ok1aqazA\nGCqqrEiW1D7E6EB/CGiqqCeqDNvpGN2ad17onNMTX7NVKxoNmpymHs6jyHS8A/iy\nJsgE6t98oe0aXPO3FtmADz9o13X7ltwy2zMpWQyNMBayXLKBFeUYUvmFgTtWF3LV\nS7HrAoGBANhxhjIHYrsAy008UAHHebsyoteIPv2i/5uaRqgkEAVA+6lgTGBxK/Hq\nJ0WRSY+9eJAJn29Br0TO6pCbXS9PUwilFxfEs2iUvDkjYjpR/p3FMNfuzHf0eqoz\nCbCL2YiGWojMIyPkfhOn/vGeE8O7qAJ2xsNFARXes5tYY8h+MmR2\n-----END
+ RSA PRIVATE KEY-----\n"}'
+ headers:
+ Accept:
+ - application/json
+ Content-Type:
+ - application/json
+ Accept-Encoding:
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+ User-Agent:
+ - Ruby
+ Host:
+ - circleci.com
+ response:
+ status:
+ code: 200
+ message: OK
+ headers:
+ Content-Type:
+ - application/json; charset=utf-8
+ Date:
+ - Sun, 19 Feb 2017 00:03:27 GMT
+ Server:
+ - nginx
+ Set-Cookie:
+ - ring-session=fy6ymRVjTcpOkNdkpyuzc0BymqpnU4K%2BCLODQLol0y8bjQdK3YmbLWmrRmCdscJJJmjx4ZdfpkXf%2BFcBrXReWvT4mCQlR5G2pzplpYQPLBOyfr3yAJWcz2nJ23T%2FOH%2BJB0i9zJB2PM3nJeZ9kTTGNpo%2FMhlai3NjQ%2BEayDB%2FsnDQ0WldVdV%2FW2Wdmbd2NL6gDwm3vZ9JmqhU42g0S0cSrq8eFzZxZVL3GOEh4Tqp8QY%3D--AQH%2B64wrEQoM%2F8bE7MtULgNcej4uth5kc2S5S7iJcxM%3D;Path=/;HttpOnly;Expires=Sun,
+ 18 Feb 2018 08:57:47 +0000;Max-Age=31536000;Secure
+ Strict-Transport-Security:
+ - max-age=15724800
+ X-Circleci-Identity:
+ - i-51a59da9
+ X-Circleci-Request-Id:
+ - 0eb0d4b8-55b0-48fd-8d39-c5c5db49968e
+ X-Frame-Options:
+ - DENY
+ X-Route:
+ - "/ssh-key"
+ Content-Length:
+ - '2'
+ Connection:
+ - keep-alive
+ body:
+ encoding: UTF-8
+ string: '""'
+ http_version:
+ recorded_at: Sat, 18 Feb 2017 23:59:39 GMT
+recorded_with: VCR 3.0.3
diff --git a/tests/fixtures/cases/tricky_secrets.min.js b/tests/fixtures/cases/tricky_secrets.min.js
new file mode 100644
index 0000000..4dec06e
--- /dev/null
+++ b/tests/fixtures/cases/tricky_secrets.min.js
@@ -0,0 +1,34 @@
+App.MODE_GOOGLE="google";App.MODE_DROPBOX="dropbox";App.MODE_ONEDRIVE="onedrive";App.MODE_GITHUB="github";App.MODE_DEVICE="device";App.MODE_BROWSER="browser";App.MODE_TRELLO="trello";App.DROPBOX_APPKEY="olezov2id9vcgml";App.DROPBOX_URL="https://unpkg.com/dropbox/dist/Dropbox-sdk.min.js";App.DROPINS_URL="https://www.dropbox.com/static/api/2/dropins.js";App.ONEDRIVE_URL="https://js.live.net/v7.2/OneDrive.js";App.TRELLO_URL="https://api.trello.com/1/client.js";App.TRELLO_JQUERY_URL="https://code.jquery.com/jquery-1.7.1.min.js";
+App.FOOTER_PLUGIN_URL="https://www.jgraph.com/drawio-footer.js";
+Wa={Cisco_cisco_telecommuter_house_pc:"shape=mxgraph.cisco.buildings.telecommuter_house_pc;fillColor=#036897;strokeColor=#ffffff",
+Cisco_cisco_telecommuter_house:"shape=mxgraph.cisco.buildings.telecommuter_house;fillColor=#036897;strokeColor=#ffffff",Cisco_cisco_telecommuter_icon:"shape=mxgraph.cisco.misc.telecommuter_icon;fillColor=#036897;strokeColor=#ffffff",Cisco_cisco_Telepresence_3200:"shape=mxgraph.cisco.misc.telepresence;fillColor=#036897;strokeColor=#ffffff",Cisco_cisco_terminal:"shape=mxgraph.cisco.computers_and_peripherals.terminal;fillColor=#036897;strokeColor=#ffffff",Cisco_cisco_token:"shape=mxgraph.cisco.misc.token;strokeColor=#036897",
+Cisco_cisco_TP_MCU:"shape=mxgraph.cisco.misc.tp_mcu;fillColor=#036897;strokeColor=#ffffff"}
+
+PageSetupDialog.getFormats = function () {
+ return [{ key: "letter", title: 'US-Letter (8,5" x 11")', format: mxConstants.PAGE_FORMAT_LETTER_PORTRAIT }, { key: "legal", title: 'US-Legal (8,5" x 14")', format: new mxRectangle(0, 0, 850, 1400) }, { key: "tabloid", title: 'US-Tabloid (11" x 17")', format: new mxRectangle(0, 0, 1100, 1700) }, { key: "executive", title: 'US-Executive (7" x 10")', format: new mxRectangle(0, 0, 700, 1E3) }, { key: "a0", title: "A0 (841 mm x 1189 mm)", format: new mxRectangle(0, 0, 3300, 4681) }, {
+ key: "a1", title: "A1 (594 mm x 841 mm)",
+ format: new mxRectangle(0, 0, 2339, 3300)
+ }, { key: "a2", title: "A2 (420 mm x 594 mm)", format: new mxRectangle(0, 0, 1654, 2336) }, { key: "a3", title: "A3 (297 mm x 420 mm)", format: new mxRectangle(0, 0, 1169, 1654) }, { key: "a4", title: "A4 (210 mm x 297 mm)", format: mxConstants.PAGE_FORMAT_A4_PORTRAIT }, { key: "a5", title: "A5 (148 mm x 210 mm)", format: new mxRectangle(0, 0, 583, 827) }, { key: "a6", title: "A6 (105 mm x 148 mm)", format: new mxRectangle(0, 0, 413, 583) }, { key: "a7", title: "A7 (74 mm x 105 mm)", format: new mxRectangle(0, 0, 291, 413) },
+ { key: "b4", title: "B4 (250 mm x 353 mm)", format: new mxRectangle(0, 0, 980, 1390) }, { key: "b5", title: "B5 (176 mm x 250 mm)", format: new mxRectangle(0, 0, 690, 980) }, { key: "16-9", title: "16:9 (1600 x 900)", format: new mxRectangle(0, 0, 1600, 900) }, { key: "16-10", title: "16:10 (1920 x 1200)", format: new mxRectangle(0, 0, 1920, 1200) }, { key: "4-3", title: "4:3 (1600 x 1200)", format: new mxRectangle(0, 0, 1600, 1200) }, { key: "custom", title: mxResources.get("custom"), format: null }]
+};
+
+t112.default.createElement("div",
+ {
+ key: "remove-button",
+ className: "form-nested-item-remove",
+ algolia:{
+ appId: "RK0UG797F3",
+ apiKey: "39d7eb90d8b31d464e309375a52d674f",
+ indexName: "datahubproject",
+ contextualSearch:!1
+ }
+ }
+)
+
+qm.staticData.appSettings ={
+ "appDesign": {
+ "clientSecret": "TcQArZOoUecO9O4aBvntUl6v1QzzsU38",
+ "privateConfig": {
+ "bugsnag_key": "ae7bc49d1285848342342bb5c321a2cf",
+ "FOURSQUARE_CLIENT_SECRET": "VUPPYPJYJQO04XYNG5DSDJZR2BQ0XWW5OWCEEYTFOOKOY1PY",
+ }}}
\ No newline at end of file
diff --git a/tests/fixtures/cheap_var_detector_cases.txt b/tests/fixtures/cheap_var_detector_cases.txt
new file mode 100644
index 0000000..6cbd93f
--- /dev/null
+++ b/tests/fixtures/cheap_var_detector_cases.txt
@@ -0,0 +1,20 @@
+client_secret: '1846d428b5340812b76c9637eceaee979340b922'
+client_secret = '2846d428b5340812b76c9637eceaee979340b922'
+
+"client_secret": "3846d428b5340812b76c9637eceaee979340b922"
+'client_secret'= '4846d428b5340812b76c9637eceaee979340b922'
+
+https://www.youtube.com/watch?v=HHi19zvnyGM&key=5846d428b5340812b76c9637eceaee979340b922
+https://www.youtube.com/watch?key=6846d428b5340812b76c9637eceaee979340b922
+
+# bitly_token <- bitly_auth(key = "7kjfkjdsabfnladksjfdsalkfmdklsa", secret = "8njdabfdsajkfndsafmhbfjdasbfndsa")
+# saveRDS(bitly_token, file = "tests/bitly_local_token.rds")
+
+
+
+
+https://www.youtube.com/watch?key=9846d428b5340812b76c9637eceaee979340b922&anotherSecretKey=10fdaskfdsajhfd5846d428b5340812b76c9637eceaee979340b922&andOnceAgainPassword=fmdksafbdsjafbdksjafds
+
+
+#Sane Reports
+[](https://blabla.com/gh/fdsa/sane-reports)
diff --git a/tests/fixtures/excluded_paths.json b/tests/fixtures/excluded_paths.json
index c770020..5fc2266 100644
--- a/tests/fixtures/excluded_paths.json
+++ b/tests/fixtures/excluded_paths.json
@@ -126,5 +126,10 @@
{
"name": "Postman collection files",
"pattern": ".*postman_collection\\.json$"
+ },
+ {
+ "name": "Deepsecrets Problem Files Fixtures Location",
+ "pattern":".*problem_files\/.*"
+
}
]
\ No newline at end of file
diff --git a/tests/fixtures/regex_checks.txt b/tests/fixtures/regex_checks.txt
index 1b7334f..29ba7d5 100644
--- a/tests/fixtures/regex_checks.txt
+++ b/tests/fixtures/regex_checks.txt
@@ -58,11 +58,39 @@ v/Ow5T0q5gIJAiEAyS4RaI9YG8EWx/2w0T67ZUVAw8eOMB6BIUg0Xcu+3okCIBOs
-----END PGP PRIVATE KEY BLOCK-----
# S19
-https://login:password@example.com
-ftp://login:password@example.com
+https://login:password@example.com # should not be found
+ftp://login:password@example.com # should not be found
amqp://login:$password@example.com
amqp://login:${password}@example.com
amqp://login:%password%@example.com
// redis://:@:
http://localhost:3001/@whatever/path/more
-http://user:sneakypass@localhost:3001/@whatever/path/more
\ No newline at end of file
+http://user:sneakypass@localhost:3001/@whatever/path/more
+
+ftp://login:testpassword@example.com # should not be found
+
+ftp://login:changeme@example.com # should not be found
+
+
+on PLAINTEXT://localhost:9092 123456terrence@igloo /usr/local/kafka15:51:25 𝜆 diff config/server.pro
+http://user:*****@localhost:3001/@whatever/path/more
+
+
+eing prematurely saved to database\n ([230e4a11](https://gitlab-ci-token:ridCNWnbTpavfVuJvWmS@git.sickrage.ca/SiCKRAGE/sickrage/commit/230e4a11b46145fc881186a3e
+
+
+5Nprr13Wue677z5uvvlmLrvsMizL4rbbbuOVr3wlhw4dItyh2t8f
+
+
+# not match
+pk_live_xxxxxxxxxxxxxxxxxxxxxxxx
+sk_live_xxxxxxxxxxxxxxxxxxxxxxxx
+sk_test_AAAAAAAAAAAAAAAAAAAAAAAA
+pk_live_YYYYYYYYYYYYYYYYYYYYYYYYYY
+
+# match
+pk_live_YYYYYYYYYYYYYfdsafdsafsd
+pk_live_sbI2q15f2zAg8Rfqiw8xmh9q
+sk_live_XCArxr2KBvWegUqwqgojiqK200qT465n
+pk_test_XCArxr2KBvWegUqwqgojiqK200qT465n
+sk_test_XCArxr2KBvWegUqwqgojiqK200qT465n
\ No newline at end of file
diff --git a/tests/generic_fixture_scans/test_run_full_scan.py b/tests/generic_fixture_scans/test_run_full_scan.py
index 01eef9f..1bb8240 100644
--- a/tests/generic_fixture_scans/test_run_full_scan.py
+++ b/tests/generic_fixture_scans/test_run_full_scan.py
@@ -1,34 +1,32 @@
-from unittest.mock import Mock
+from typing import List
+
import pytest
-from deepsecrets.config import Config, Output
-from deepsecrets.core.engines.regex import RegexEngine
-from deepsecrets.core.engines.semantic import SemanticEngine
-from deepsecrets.core.rulesets.false_findings import FalseFindingsBuilder
-from deepsecrets.core.rulesets.regex import RegexRulesetBuilder
-from deepsecrets.scan_modes.cli import CliScanMode
+from deepsecrets.cli import DeepSecretsCliTool
-@pytest.fixture()
-def config() -> Config:
- config = None
- config = Config()
- config.set_workdir('tests/fixtures')
- config.engines.append(RegexEngine)
- config.engines.append(SemanticEngine)
- config.add_ruleset(RegexRulesetBuilder, ['tests/fixtures/regexes.json'])
- config.add_ruleset(FalseFindingsBuilder, ['tests/fixtures/false_findings.json'])
- config.output = Output(type='json', path='tests/1.json')
- return config
+@pytest.fixture(scope='module')
+def args():
+ return [
+ '',
+ '--target-dir',
+ '/app/tests/fixtures/',
+ '--outfile',
+ './fdsafad.json',
+ '--outformat',
+ 'dojo-sarif',
+ '--benchmarking-mode',
+ '--process-count',
+ '1',
+ ]
-def test_everything(config: Config) -> None:
- mode = CliScanMode(config=config)
- mode.progress_bar = Mock()
- mode.progress_bar.add_task.return_value = 0
- findings = mode.run()
+def test_everything(args: List[str]) -> None:
+ tool = DeepSecretsCliTool(args)
+ tool.parse_arguments()
+ findings, errors, timings, _ = tool.start()
detections = [finding.detection for finding in findings]
assert 'bAicxJVa5uVY7MjDlapthw' in detections
assert 'nacc6opq' in detections
- assert 'xBfiGBARuoQ9HoLWtw1HwbrkPurCI8v7fO7RJDaZFp7gkBqWxRjQc9WemTVrwu1c' in detections
\ No newline at end of file
+ assert 'xBfiGBARuoQ9HoLWtw1HwbrkPurCI8v7fO7RJDaZFp7gkBqWxRjQc9WemTVrwu1c' in detections
diff --git a/tests/output/test_sarif.py b/tests/output/test_sarif.py
index 718fca2..a8c8ac0 100644
--- a/tests/output/test_sarif.py
+++ b/tests/output/test_sarif.py
@@ -1,16 +1,18 @@
from unittest.mock import Mock
+from jschema_to_python.to_json import to_json
import pytest
from deepsecrets.config import Config, Output
from deepsecrets.core.engines.regex import RegexEngine
from deepsecrets.core.engines.semantic import SemanticEngine
-from deepsecrets.core.model.finding import FindingResponse
+from deepsecrets.core.model.response.dojo_sarif import DojoSarifResponseBuilder
from deepsecrets.core.rulesets.false_findings import FalseFindingsBuilder
from deepsecrets.core.rulesets.regex import RegexRulesetBuilder
from deepsecrets.scan_modes.cli import CliScanMode
FP_TO_BE_EXCLUDED = '/app/tests/fixtures/service.postman_collection.json'
+
@pytest.fixture()
def config() -> Config:
config = None
@@ -28,13 +30,25 @@ def test_dojo_sarif(config: Config) -> None:
mode = CliScanMode(config=config)
mode.progress_bar = Mock()
mode.progress_bar.add_task.return_value = 0
+ mode.progress_bar.task_ids = []
findings = []
+
for file in mode.filepaths:
- findings.extend(mode._per_file_analyzer(mode.analyzer_bundle(), file))
+ findings.extend(mode._per_file_analyzer(mode.analyzer_bundle(), file, 0, {}).findings)
+ '''
+ # checking through the 'run' method
+ # false findings checked at the end
findings = []
findings = mode.run()
-
- sarif_response = FindingResponse.dojo_sarif_from_list(findings)
- assert sarif_response is not None
\ No newline at end of file
+ '''
+
+ sarif_response = to_json(
+ DojoSarifResponseBuilder()
+ .with_current_mode(mode)
+ .with_findings_list(findings)
+ .with_masking_enabled(not config.disable_masking)
+ .build()
+ )
+ assert sarif_response is not None
diff --git a/tests/scan_modes/test_cli_scan_mode.py b/tests/scan_modes/test_cli_scan_mode.py
index 860a3f2..aa37906 100644
--- a/tests/scan_modes/test_cli_scan_mode.py
+++ b/tests/scan_modes/test_cli_scan_mode.py
@@ -9,12 +9,14 @@
FP_TO_BE_EXCLUDED = '/app/tests/fixtures/service.postman_collection.json'
+
@pytest.fixture()
def config() -> Config:
config = None
config = Config()
config.set_workdir('tests/fixtures')
config.engines.append(RegexEngine)
+
config.add_ruleset(RegexRulesetBuilder, ['tests/fixtures/regexes.json'])
config.add_ruleset(FalseFindingsBuilder, ['tests/fixtures/false_findings.json'])
config.output = Output(type='json', path='tests/1.json')
@@ -31,16 +33,17 @@ def test_cli_scan_mode(config: Config) -> None:
mode.progress_bar = Mock()
mode.progress_bar.add_task.return_value = 0
+ mode.progress_bar.task_ids = []
findings = []
for file in mode.filepaths:
- findings.extend(mode._per_file_analyzer(mode.analyzer_bundle(), file))
+ findings.extend(mode._per_file_analyzer(mode.analyzer_bundle(), file, 0, {}).findings)
- assert len(findings) == 3
+ assert len(findings) == 6
# checking through the 'run' method
# false findings checked at the end
findings = []
findings = mode.run()
- assert len(findings) == 2
+ assert len(findings) == 3