diff --git a/docs/getting_started.md b/docs/getting_started.md
index cc027db..794d5d6 100644
--- a/docs/getting_started.md
+++ b/docs/getting_started.md
@@ -52,7 +52,11 @@ Now on the server where you wish to run the chatbot, start the chatbot itself:
 make serve    # Runs inside of docker for ease of use
 ```
 
-Now download the kssh binary and start SSHing! See https://github.com/keybase/bot-sshca/releases to download the most 
+You can confirm that the bot is running by sending the message `ping @bot_username` in any of the configured team chat 
+channels (if `CHAT_CHANNEL` is configured, the message must be sent in that specific channel). The bot should reply with 
+`pong @your_username`. 
+
+Now you can download the kssh binary and start SSHing! See https://github.com/keybase/bot-sshca/releases to download the most 
 recent version of kssh for your platform. 
 
 ```bash
@@ -72,4 +76,3 @@ If you update any environment variables, it is necessary to restart the keybasec
 by running `make restart`. Note that it is not required to re-run `make generate`. 
 
 Note that this means `kssh` will not work for a brief period of time while the container restarts.
-
diff --git a/src/keybaseca/bot/bot.go b/src/keybaseca/bot/bot.go
index c575ae7..627a691 100644
--- a/src/keybaseca/bot/bot.go
+++ b/src/keybaseca/bot/bot.go
@@ -88,7 +88,7 @@ func StartBot(conf config.Config) error {
 		if shared.IsPingRequest(messageBody, kbc.GetUsername()) {
 			// Respond to messages of the form `ping @botName` with `pong @senderName`
 			log.Debug("Responding to ping with pong")
-			_, err = kbc.SendMessageByConvID(msg.Message.ConvID, fmt.Sprintf(shared.GeneratePingResponse(msg.Message.Sender.Username)))
+			_, err = kbc.SendMessageByConvID(msg.Message.ConvID, shared.GeneratePingResponse(msg.Message.Sender.Username))
 			if err != nil {
 				LogError(conf, kbc, msg, err)
 				continue
diff --git a/src/shared/chat_types.go b/src/shared/chat_types.go
index 197a120..730a6d8 100644
--- a/src/shared/chat_types.go
+++ b/src/shared/chat_types.go
@@ -82,21 +82,21 @@ func IsAckResponse(msg string) bool {
 }
 
 // Generate a ping request message
-func GeneratePingRequest(botname string) string {
-	return fmt.Sprintf("ping @%s", botname)
+func GeneratePingRequest(username string) string {
+	return fmt.Sprintf("ping @%s", username)
 }
 
 // Returns whether the given message is a ping request
 func IsPingRequest(msg, botUsername string) bool {
-	return strings.TrimSpace(msg) == fmt.Sprintf("ping @%s", botUsername)
+	return strings.TrimSpace(msg) == GeneratePingRequest(botUsername)
 }
 
 // Generate a ping response message
-func GeneratePingResponse(botname string) string {
-	return fmt.Sprintf("pong @%s", botname)
+func GeneratePingResponse(username string) string {
+	return fmt.Sprintf("pong @%s", username)
 }
 
 // Returns whether the given message is a ping response
 func IsPingResponse(msg, localUsername string) bool {
-	return strings.TrimSpace(msg) == fmt.Sprintf("pong @%s", localUsername)
+	return strings.TrimSpace(msg) == GeneratePingResponse(localUsername)
 }
diff --git a/tests/bot-entrypoint.py b/tests/bot-entrypoint.py
index 5344d2d..1e55a58 100644
--- a/tests/bot-entrypoint.py
+++ b/tests/bot-entrypoint.py
@@ -28,10 +28,10 @@ def load_env():
         "echo yes | bin/keybaseca backup > /shared/cakey.backup\n"
         # The output from this sign operation is tested in test_env_1.py
         "ssh-keygen -t ed25519 -f /shared/userkey -N '' && bin/keybaseca sign --public-key /shared/userkey.pub > /shared/keybaseca-sign.out\n"
-        "bin/keybaseca service &"
+        "bin/keybaseca --debug service > /tmp/debug.out &"
     ) % (shlex.quote(path)))
     # Sleep so keybaseca has time to start
-    time.sleep(5)
+    time.sleep(10)
     return "OK"
 
 if __name__ == '__main__':
diff --git a/tests/tests/lib.py b/tests/tests/lib.py
index bd1f565..20f194b 100644
--- a/tests/tests/lib.py
+++ b/tests/tests/lib.py
@@ -127,7 +127,7 @@ def outputs_audit_log(tc: TestConfig, filename: str, expected_number: int):
     yield
 
     # And sleep to give KBFS some time
-    time.sleep(1.5)
+    time.sleep(2.5)
 
     # Then see if there are new lines using set difference. This is only safe/reasonable since we include a
     # timestamp in audit log lines.
diff --git a/tests/tests/test_env_1.py b/tests/tests/test_env_1.py
index 7ce1a35..10f694e 100644
--- a/tests/tests/test_env_1.py
+++ b/tests/tests/test_env_1.py
@@ -16,6 +16,12 @@ def configure_env(self):
     def test_config(self):
         return TestConfig.getDefaultTestConfig()
 
+    def test_ping_pong_command(self, test_config):
+        run_command(f"keybase chat send --channel ssh-provision {test_config.subteam}.ssh 'ping @{test_config.bot_username}'")
+        time.sleep(5)
+        recent_messages = run_command(f"keybase chat list-unread --since 1m")
+        assert (b"pong @%s" % test_config.username.encode('utf-8')) in recent_messages
+
     def test_kssh_staging_user(self, test_config):
         # Test ksshing into staging as user
         with outputs_audit_log(test_config, filename=test_env_1_log_filename, expected_number=1):
@@ -166,4 +172,4 @@ def test_kssh_alternate_binary(self, test_config):
             assert_contains_hash(test_config.expected_hash, run_command_with_agent("bin/kssh -q -o StrictHostKeyChecking=no user@sshd-staging 'sha1sum /etc/unique'"))
             run_command("bin/kssh --set-keybase-binary ''")
         finally:
-            run_command("sudo rm /usr/local/bin/keybase")
\ No newline at end of file
+            run_command("sudo rm /usr/local/bin/keybase")