Skip to content

Commit b576ec5

Browse files
vlussenburgunionfindbee
vlussenburg
and
unionfindbee
authored
Add coverage junit input runid output (#45)
* Added `junit-output`, `coverage-output` * added logic in main for junit + coverage * added ci for the mcode action * added test for new functionality * Add Mayhemfile for testing * added run output * Create dependabot.yml --------- Co-authored-by: Benjamin Gutierrez <ben.gutierrez@forallsecure.com>
1 parent b739332 commit b576ec5

14 files changed

Lines changed: 5342 additions & 5631 deletions

File tree

‎.github/dependabot.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
version: 2
2+
updates:
3+
# Enable version updates for npm
4+
- package-ecosystem: 'npm'
5+
# Look for `package.json` and `lock` files in the `root` directory
6+
directory: '/'
7+
# Check the npm registry for updates every day (weekdays)
8+
schedule:
9+
interval: 'daily'

‎.github/workflows/main.yml‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
name: 'build-test'
2+
on:
3+
push
4+
5+
jobs:
6+
build: # make sure build/ci work properly
7+
runs-on: ubuntu-latest
8+
env:
9+
MAYHEM_TOKEN: ${{ secrets.MAYHEM_TOKEN }}
10+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
11+
steps:
12+
- uses: actions/checkout@v3
13+
with:
14+
# fetch entire history to compute diffs between jobs
15+
fetch-depth: 0
16+
- run: |
17+
npm install
18+
- run: |
19+
npm run all
20+
21+
test-some-outputs: # make sure the action works on a clean machine without building
22+
runs-on: ubuntu-latest
23+
steps:
24+
- uses: actions/checkout@v3
25+
with:
26+
# fetch entire history to compute diffs between jobs
27+
fetch-depth: 0
28+
29+
- uses: ./
30+
id: mcode-action
31+
with:
32+
mayhem-url: https://demo.forallsecure.com
33+
mayhem-token: ${{ secrets.MAYHEM_TOKEN }}
34+
github-token: ${{ secrets.GITHUB_TOKEN }}
35+
sarif-output: out/sarif/
36+
args: --image forallsecure/lighttpd:vulnerable --file __tests__/Mayhemfile --duration 60
37+
38+
- name: Upload SARIF file(s)
39+
uses: github/codeql-action/upload-sarif@v2
40+
with:
41+
sarif_file: out/sarif
42+
43+
- name: Print runId (${{ steps.mcode-action.outputs.runId }}) and test it's non-empty
44+
run: |
45+
[ -z "${{ steps.mcode-action.outputs.runId }}" ] && echo "runId was blank!" && exit 1;
46+
echo "The run id was: ${{ steps.mcode-action.outputs.runId }}"
47+
48+
test-all-outputs: # make sure the action works on a clean machine without building
49+
runs-on: ubuntu-latest
50+
steps:
51+
- uses: actions/checkout@v3
52+
with:
53+
# fetch entire history to compute diffs between jobs
54+
fetch-depth: 0
55+
56+
- uses: ./
57+
id: mcode-action
58+
with:
59+
mayhem-url: https://demo.forallsecure.com
60+
mayhem-token: ${{ secrets.MAYHEM_TOKEN }}
61+
github-token: ${{ secrets.GITHUB_TOKEN }}
62+
junit-output: out/junit/
63+
sarif-output: out/sarif/
64+
coverage-output: out/coverage/
65+
args: --image forallsecure/lighttpd:vulnerable --file __tests__/Mayhemfile --duration 60
66+
67+
- name: Archive Coverage report
68+
uses: actions/upload-artifact@v3
69+
with:
70+
name: coverage-report
71+
path: out/coverage/
72+
73+
- name: Archive JUnit results
74+
uses: actions/upload-artifact@v3
75+
with:
76+
name: mcode-junit
77+
path: out/junit/
78+
79+
- name: Upload SARIF file(s)
80+
uses: github/codeql-action/upload-sarif@v2
81+
with:
82+
sarif_file: out/sarif
83+
84+
- name: Print runId (${{ steps.mcode-action.outputs.runId }}) and test it's non-empty
85+
run: |
86+
[ -z "${{ steps.mcode-action.outputs.runId }}" ] && echo "runId was blank!" && exit 1;
87+
echo "The run id was: ${{ steps.mcode-action.outputs.runId }}"
88+
89+
test-no-outputs: # make sure the action works on a clean machine without building
90+
runs-on: ubuntu-latest
91+
steps:
92+
- uses: actions/checkout@v3
93+
with:
94+
# fetch entire history to compute diffs between jobs
95+
fetch-depth: 0
96+
97+
- uses: ./
98+
id: mcode-action
99+
with:
100+
mayhem-url: https://demo.forallsecure.com
101+
mayhem-token: ${{ secrets.MAYHEM_TOKEN }}
102+
github-token: ${{ secrets.GITHUB_TOKEN }}
103+
args: --image forallsecure/lighttpd:vulnerable --file __tests__/Mayhemfile --duration 60
104+
105+
- name: Print runId (${{ steps.mcode-action.outputs.runId }}) and test it's non-empty
106+
run: |
107+
[ -z "${{ steps.mcode-action.outputs.runId }}" ] && echo "runId was blank!" && exit 1;
108+
echo "The run id was: ${{ steps.mcode-action.outputs.runId }}"

‎README.md‎

Lines changed: 28 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -122,11 +122,30 @@ jobs:
122122
mayhem-token: ${{ secrets.MAYHEM_TOKEN }}
123123
args: --image ${{ needs.build.outputs.image }} --file ${{ matrix.mayhemfile }} --duration 300
124124
sarif-output: sarif
125+
junit-output: junit
126+
coverage-output: coverage
125127
126128
- name: Upload SARIF file(s)
127129
uses: github/codeql-action/upload-sarif@v2
128130
with:
129131
sarif_file: sarif
132+
133+
- name: Archive Coverage report
134+
uses: actions/upload-artifact@v3
135+
with:
136+
name: coverage-report
137+
path: coverage
138+
139+
- name: Archive JUnit results
140+
uses: actions/upload-artifact@v3
141+
with:
142+
name: mcode-junit
143+
path: junit
144+
145+
- name: Upload SARIF file(s)
146+
uses: github/codeql-action/upload-sarif@v2
147+
with:
148+
sarif_file: sarif
130149
```
131150

132151
The mCode Action accepts the following inputs:
@@ -137,6 +156,14 @@ The mCode Action accepts the following inputs:
137156
| | `mayhem-token` | string | Mayhem for Code account token. **Only required within** `mayhem.yml` **if overriding** `mayhem-url`. |
138157
| | `args` | string | Additional CLI override [arguments](https://mayhem.forallsecure.com/docs/mayhem-cli/getting-started/mayhem-cli-commands/#run) such as specifying the `--testsuite` directory path for a seed test suite. |
139158
| | `sarif-output` | string | Path for generating a SARIF report output file. |
159+
| | `junit-output` | string | Path for generating a jUnit report output file. |
160+
| | `coverage-output` | string | Path for generating a coverage report output files. |
161+
162+
The mCode Action provides the following outputs:
163+
| Output Name | Type | Description | Default
164+
| --- | --- | --- | ---
165+
| `runId` | string | The identifier of the run that this action triggered in Mayhem. |
166+
140167

141168
📖 See the [CI/CD](https://mayhem.forallsecure.com/docs/mayhem-ci-cd/fuzzing-in-your-pipeline/) docs for more information and guides on using the mCode GitHub Action!
142169

@@ -146,14 +173,7 @@ Mayhem for Code generates [SARIF reports](https://sarifweb.azurewebsites.net/#:~
146173

147174
SARIF reports are generated using the `sarif-output` parameter, which specifies an output file path.
148175

149-
To upload the SARIF report to GitHub, use the `github/codeql-action/upload-sarif@v2` action with the `sarif_file` parameter to specify the location of a path containing SARIF results to upload to GitHub.
150-
151-
```yaml
152-
- name: Upload SARIF file(s)
153-
uses: github/codeql-action/upload-sarif@v2
154-
with:
155-
sarif_file: sarif
156-
```
176+
To upload the SARIF report to GitHub, see the `Upload SARIF file(s)` step in the `mayhem.yml` example above.
157177

158178
Once uploaded to GitHub, you can view test results in the `Security` tab of your repository as well as for your individual pull requests.
159179

‎__tests__/Mayhemfile‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
image: forallsecure/lighttpd:vulnerable # fields are unnecessary
2+
duration: 60 # since they will
3+
project: forallsecure/lighttpd # be filled at
4+
target: lighttpd # run creation time
5+
advanced_triage: true
6+
tasks:
7+
- name: exploitability_factors
8+
- name: regression_testing
9+
- name: behavior_testing
10+
- name: coverage_analysis
11+
cmds:
12+
- cmd: /usr/local/sbin/lighttpd -D -f /usr/local/etc/lighttpd.conf
13+
network:
14+
url: tcp://localhost:80
15+
timeout: 2
16+
client: false

‎__tests__/events.json‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{
2+
"pull_request": {
3+
"head": {
4+
"ref": "sample-head-ref",
5+
"sha": "abcdef1234567890"
6+
},
7+
"base": {
8+
"ref": "sample-base-ref"
9+
}
10+
}
11+
}

‎__tests__/main.test.ts‎

Lines changed: 28 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,46 @@
11
import * as process from "process";
22
import { ExecFileSyncOptions, execFileSync } from "child_process";
33
import * as path from "path";
4+
import fs from "fs";
45

56
// shows how the runner will run a javascript action with env / stdout protocol
67
test("test runs", () => {
7-
process.env["GITHUB_REPOSITORY"] = "ForAllSecure/mapi-action";
8+
process.env["GITHUB_REPOSITORY"] = "ForAllSecure/mcode-action";
9+
process.env["GITHUB_SERVER_URL"] = "https://github.com";
10+
process.env["GITHUB_RUN_ID"] = "14";
11+
process.env["GITHUB_EVENT_PATH"] = "__tests__/events.json";
812
process.env["RUNNER_TEMP"] = "/tmp";
913
process.env["RUNNER_TOOL_CACHE"] = "/tmp";
10-
process.env["INPUT_MAPI-TOKEN"] = process.env.MAPI_TOKEN;
14+
15+
process.env["INPUT_MAYHEM-TOKEN"] = process.env.MAYHEM_TOKEN;
1116
process.env["INPUT_DURATION"] = "10";
12-
process.env["INPUT_API-URL"] =
13-
"https://demo-api.mayhem4api.forallsecure.com/api/v3";
14-
process.env["INPUT_API-SPEC"] =
15-
"https://demo-api.mayhem4api.forallsecure.com/api/v3/openapi.json";
17+
process.env["INPUT_GITHUB-TOKEN"] = "12123123321312";
18+
19+
process.env["INPUT_JUNIT-OUTPUT"] = "junit-output";
20+
process.env["INPUT_SARIF-OUTPUT"] = "sarif-output";
21+
process.env["INPUT_COVERAGE-OUTPUT"] = "coverage-output";
22+
1623
const np = process.execPath;
1724
const ip = path.join(__dirname, "..", "lib", "main.js");
1825
const options: ExecFileSyncOptions = {
1926
env: process.env,
2027
};
2128
try {
22-
execFileSync(np, [ip], options);
23-
throw new Error("Actions should have failed due to finding a bug");
29+
console.log(execFileSync(np, [ip], options).toString());
2430
} catch (error: any) {
25-
if (error.stdout === undefined) {
26-
throw error;
27-
}
28-
expect(error.stdout.toString()).toEqual(
29-
expect.stringContaining("Fuzzing complete!,")
30-
);
31-
expect(error.stdout.toString()).toEqual(
32-
expect.stringContaining("Found internal-server-error!")
33-
);
34-
expect(error.stdout.toString()).toEqual(
35-
expect.stringContaining("The Mayhem for API scan found issues in the API")
36-
);
31+
// Ignore the error. We known the Mayhemfile doesn't exists right now, so the bash script fails.
32+
//console.log(error);
33+
}
34+
35+
if (!fs.existsSync("junit-output")) {
36+
throw new Error("Output dir should exist but didn't");
37+
}
38+
39+
if (!fs.existsSync("sarif-output")) {
40+
throw new Error("Output dir should exist but didn't");
41+
}
42+
43+
if (!fs.existsSync("coverage-output")) {
44+
throw new Error("Output dir should exist but didn't");
3745
}
3846
});

‎action.yml‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,13 @@ inputs:
1313
required: false
1414
default: ${{ github.token }}
1515
sarif-output:
16-
description: SARIF output directory. You can upload to to GitHub using the 'github/codeql-action/upload-sarif@v1' action
16+
description: SARIF output path (must be a directory, doesn't have to exist yet). You can upload to to GitHub using the 'github/codeql-action/upload-sarif@v1' action
17+
required: false
18+
junit-output:
19+
description: jUnit output path (must be a directory, doesn't have to exist yet). You can upload the artifact to GitHub using the 'actions/upload-artifact' action
20+
required: false
21+
coverage-output:
22+
description: coverage report output path (must be a directory, doesn't have to exist yet). You can upload the artifacts to GitHub using the 'actions/upload-artifact' action
1723
required: false
1824
verbosity:
1925
description: verbosity level for starting runs
@@ -24,6 +30,9 @@ inputs:
2430
runs:
2531
using: "node16"
2632
main: "dist/index.js"
33+
outputs:
34+
runId:
35+
description: The identifier of the run that this action triggered in Mayhem
2736
branding:
2837
icon: "shield"
2938
color: "red"

0 commit comments

Comments
 (0)