diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml
index 55bbbae..d67e59e 100644
--- a/.github/workflows/on-push.yml
+++ b/.github/workflows/on-push.yml
@@ -16,7 +16,7 @@ jobs:
       - name: Install MultiTool via Action
         uses: "./"
         with:
-          install-only: true
+          install-only: "true"
           version: "v0.3.0"
 
       - name: "Test CLI is installed"
@@ -40,5 +40,5 @@ jobs:
         with:
           email: ${{ secrets.MULTITOOL_QA_EMAIL }}
           password: ${{ secrets.MULTITOOL_QA_PASSWORD }}
-          dry-run: true
-          cache-binary: false
+          dry-run: "true"
+          cache-binary: "false"
diff --git a/action.yml b/action.yml
index 876ea75..efbf834 100644
--- a/action.yml
+++ b/action.yml
@@ -15,12 +15,12 @@ inputs:
   install-only:
     description: "Whether to install the CLI and then exit"
     required: false
-    default: false
+    default: "false"
 
   dry-run:
     description: "Skips running `multi run`. Useful for testing login credentials."
     required: false
-    default: false
+    default: "false"
 
   email:
     description: "An email address to use when logging in."
@@ -33,7 +33,7 @@ inputs:
   cache-binary:
     description: "When enabled (default), the `multi` binary will be added to the Actions cache and reused between runs"
     required: false
-    default: true
+    default: "true"
 
 outputs:
   output-string:
@@ -52,8 +52,10 @@ runs:
 
     - name: Login
       shell: bash
+      if: ${{ inputs.install-only != 'true' }}
       run: "multi login --enable-colors=always --email=${{ inputs.email }} --password=${{ inputs.password }}"
 
     - name: Run
       shell: bash
+      if: ${{ inputs.install-only != 'true' && inputs.dry-run != 'true' }}
       run: "multi run --enable-colors=always"
\ No newline at end of file