Add CI test pipeline with Key Vault secrets and log redaction

.gitignore

@@ -330,4 +330,9 @@ ASALocalRun/
 .mfractor/
 
 # temp files
-tmp_*
+tmp_*
+
+# Test secrets (generated by CI or local dev)
+secrets/
+test.subscriptions.regions.json
+test.certificates.json

ci/azure-pipelines.yml

@@ -59,3 +59,15 @@ steps:
     export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$HOME/carbon/current/lib/x64"
     go build -v ./...
   displayName: 'Build'
+
+- template: generate-subscription-file.yml
+
+- script: |
+    set +x  # NEVER enable trace mode
+    export CGO_CFLAGS="-I$HOME/carbon/current/include/c_api"
+    export CGO_LDFLAGS="-L$HOME/carbon/current/lib/x64 -lMicrosoft.CognitiveServices.Speech.core"
+    export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$HOME/carbon/current/lib/x64"
+
+    source ci/load-build-secrets.sh
+    go test -v ./speech 2>&1 | global_redact
+  displayName: 'Run Go tests'

ci/generate-subscription-file.yml

@@ -0,0 +1,17 @@
+# Copyright (c) Microsoft. All rights reserved.
+# Licensed under the MIT license. See LICENSE.md file in the project root for full license information.
+#
+# This file defines a reusable step that downloads the subscriptions JSON from Azure Key Vault
+# and writes it to a local file for use by load-build-secrets.sh.
+steps:
+- task: AzureKeyVault@2
+  inputs:
+    azureSubscription: 'ADO -> Speech Services - DEV - SDK'
+    keyVaultName: "CarbonSDK-CICD"
+    secretsFilter: 'CarbonSubscriptionsJson'
+- task: file-creator@6
+  inputs:
+    filepath: '$(Build.SourcesDirectory)/secrets/test.subscriptions.regions.json'
+    filecontent: '$(CarbonSubscriptionsJson)'
+    fileoverwrite: true
+  displayName: "Ensure subscriptions .json file"

ci/load-build-secrets.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+# Copyright (c) Microsoft. All rights reserved.
+# Licensed under the MIT license. See LICENSE.md file in the project root for full license information.
+
+set -euo pipefail
+set +x  # Never trace secrets
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+SECRETS_DIR="${BUILD_SOURCESDIRECTORY:-$REPO_ROOT}/secrets"
+SUBSCRIPTIONS_FILE="$SECRETS_DIR/test.subscriptions.regions.json"
+
+if [[ ! -f "$SUBSCRIPTIONS_FILE" ]]; then
+  echo "ERROR: Subscriptions JSON not found at $SUBSCRIPTIONS_FILE"
+  exit 1
+fi
+
+if ! command -v jq &>/dev/null; then
+  echo "ERROR: jq is required but not installed."
+  exit 1
+fi
+
+export SPEECH_SUBSCRIPTION_KEY=$(jq -jr '.UnifiedSpeechSubscription.Key' "$SUBSCRIPTIONS_FILE")
+export SPEECH_SUBSCRIPTION_REGION=$(jq -jr '.UnifiedSpeechSubscription.Region' "$SUBSCRIPTIONS_FILE")
+
+if [[ -z "$SPEECH_SUBSCRIPTION_KEY" || "$SPEECH_SUBSCRIPTION_KEY" == "null" ]]; then
+  echo "ERROR: Failed to extract UnifiedSpeechSubscription.Key from $SUBSCRIPTIONS_FILE"
+  exit 1
+fi
+
+if [[ -z "$SPEECH_SUBSCRIPTION_REGION" || "$SPEECH_SUBSCRIPTION_REGION" == "null" ]]; then
+  echo "ERROR: Failed to extract UnifiedSpeechSubscription.Region from $SUBSCRIPTIONS_FILE"
+  exit 1
+fi
+
+echo "Loaded speech subscription for region: $SPEECH_SUBSCRIPTION_REGION"
+
+GLOBAL_STRINGS_TO_REDACT=(
+  "$SPEECH_SUBSCRIPTION_KEY"
+)
+
+URL_ENCODED_SPEECH_SUBSCRIPTION_KEY=$(jq -nr --arg v "$SPEECH_SUBSCRIPTION_KEY" '$v | @uri')
+if [[ -n "$URL_ENCODED_SPEECH_SUBSCRIPTION_KEY" && "$URL_ENCODED_SPEECH_SUBSCRIPTION_KEY" != "$SPEECH_SUBSCRIPTION_KEY" ]]; then
+  GLOBAL_STRINGS_TO_REDACT+=("$URL_ENCODED_SPEECH_SUBSCRIPTION_KEY")
+fi
+unset URL_ENCODED_SPEECH_SUBSCRIPTION_KEY
+
+redact_input_with() {
+  perl -MIO::Handle -lpe \
+    'BEGIN {
+       STDOUT->autoflush(1);
+       STDERR->autoflush(1);
+       if (@ARGV) {
+         $re = sprintf "(?:%s)", (join "|", map { quotemeta $_ } splice @ARGV);
+         $re = qr/$re/
+       }
+     }
+     $re and s/$re/***/gi' "$@"
+}
+
+global_redact() {
+  redact_input_with "${GLOBAL_STRINGS_TO_REDACT[@]}"
+}
+
+export -f redact_input_with
+export -f global_redact

speech/conversation_transcriber_test.go

@@ -30,7 +30,7 @@ func setupConversation(t *testing.T) (teardown func()) {
 				logLines.WriteString(diagnostics.GetMemoryLogLine(i))
 			}
 
-			t.Log(logLines.String())
+			t.Log(redactSecrets(logLines.String()))
 		}
 	}
 }
@@ -222,14 +222,14 @@ func TestConversationTranscriberSingleSpeaker(t *testing.T) {
 
 	transcribingFuture := make(chan bool)
 	transcribedFuture := make(chan bool)
-	
+
 	transcribedHandler := func(event ConversationTranscriptionEventArgs) {
 		defer event.Close()
 		t.Log("Transcribed text: ", event.Result.Text)
 		t.Log("Speaker ID: ", event.Result.SpeakerID)
 		transcribedFuture <- true
 	}
-	
+
 	transcribingHandler := func(event ConversationTranscriptionEventArgs) {
 		defer event.Close()
 		t.Log("Transcribing text: ", event.Result.Text)
@@ -239,32 +239,32 @@ func TestConversationTranscriberSingleSpeaker(t *testing.T) {
 		default:
 		}
 	}
-	
+
 	transcriber.Transcribed(transcribedHandler)
 	transcriber.Transcribing(transcribingHandler)
-	
+
 	// Start transcribing
 	err := <-transcriber.StartTranscribingAsync()
 	if err != nil {
 		t.Error("Got error: ", err)
 	}
-	
+
 	// Wait for transcribing event
 	select {
 	case <-transcribingFuture:
 		t.Log("Received Transcribing event.")
 	case <-time.After(5 * time.Second):
 		t.Error("Timeout waiting for Transcribing event.")
 	}
-	
+
 	// Wait for transcribed event
 	select {
 	case <-transcribedFuture:
 		t.Log("Received Transcribed event.")
 	case <-time.After(5 * time.Second):
 		t.Error("Timeout waiting for Transcribed event.")
 	}
-	
+
 	// Stop transcribing
 	err = <-transcriber.StopTranscribingAsync()
 	if err != nil {
@@ -290,65 +290,65 @@ func TestConversationTranscriberContinuousRecognition(t *testing.T) {
 		t.Error("Got an error ", err.Error())
 	}
 	defer format.Close()
-	
+
 	stream, err := audio.CreatePushAudioInputStreamFromFormat(format)
 	if err != nil {
 		t.Error("Got an error ", err.Error())
 	}
 	defer stream.Close()
-	
+
 	audioConfig, err := audio.NewAudioConfigFromStreamInput(stream)
 	if err != nil {
 		t.Error("Got an error ", err.Error())
 	}
 	defer audioConfig.Close()
-	
+
 	transcriber := createConversationTranscriberFromAudioConfig(t, audioConfig)
 	if transcriber == nil {
 		t.Error("Transcriber creation failed")
 		return
 	}
 	defer transcriber.Close()
-	
+
 	firstResult := true
 	transcribedFuture := make(chan string, 10)
 	transcribingFuture := make(chan string, 10)
 	sessionStoppedFuture := make(chan bool, 1)
 	canceledFuture := make(chan bool, 1)
-	
+
 	// Channel to collect speaker IDs
 	speakerIDsChan := make(chan string, 200)
-	
+
 	transcribedHandler := func(event ConversationTranscriptionEventArgs) {
 		defer event.Close()
 		firstResult = true
 		t.Log("Transcribed: ", event.Result.Text)
 		t.Log("Speaker ID: ", event.Result.SpeakerID)
-		
+
 		// Send speaker ID to the channel if it's not empty
 		if event.Result.SpeakerID != "" && event.Result.SpeakerID != "Unknown" {
 			speakerIDsChan <- event.Result.SpeakerID
 		}
-		
+
 		transcribedFuture <- "Transcribed"
 	}
-	
+
 	transcribingHandler := func(event ConversationTranscriptionEventArgs) {
 		defer event.Close()
 		t.Log("Transcribing: ", event.Result.Text)
 		t.Log("Speaker ID: ", event.Result.SpeakerID)
-		
+
 		// Send speaker ID to the channel if it's not empty
 		if event.Result.SpeakerID != "" && event.Result.SpeakerID != "Unknown" {
 			speakerIDsChan <- event.Result.SpeakerID
 		}
-		
+
 		if firstResult {
 			firstResult = false
 			transcribingFuture <- "Transcribing"
 		}
 	}
-	
+
 	transcriber.Transcribed(transcribedHandler)
 	transcriber.Transcribing(transcribingHandler)
 	transcriber.SessionStopped(func(event SessionEventArgs) {
@@ -364,78 +364,78 @@ func TestConversationTranscriberContinuousRecognition(t *testing.T) {
 		}
 		t.Error("Canceled was not due to EOS " + event.ErrorDetails)
 	})
-	
+
 	err = <-transcriber.StartTranscribingAsync()
 	if err != nil {
 		t.Error("Got error: ", err)
 	}
-	
+
 	// Pump audio data into the stream
 	pumpFileIntoStream(t, "../test_files/katiesteve_mono.wav", stream)
 	pumpFileIntoStream(t, "../test_files/katiesteve_mono.wav", stream)
 	pumpSilenceIntoStream(t, stream)
 	stream.CloseStream()
-	
+
 	// Wait for first transcribing event
 	select {
 	case <-transcribingFuture:
 		t.Log("Received first Transcribing event.")
 	case <-time.After(30 * time.Second):
 		t.Error("Didn't receive first Transcribing event.")
 	}
-	
+
 	// Wait for first transcribed event
 	select {
 	case <-transcribedFuture:
 		t.Log("Received first Transcribed event.")
 	case <-time.After(30 * time.Second):
 		t.Error("Didn't receive first Transcribed event.")
 	}
-	
+
 	// Wait for second transcribing event
 	select {
 	case <-transcribingFuture:
 		t.Log("Received second Transcribing event.")
 	case <-time.After(30 * time.Second):
 		t.Error("Didn't receive second Transcribing event.")
 	}
-	
+
 	// Wait for second transcribed event
 	select {
 	case <-transcribedFuture:
 		t.Log("Received second Transcribed event.")
 	case <-time.After(30 * time.Second):
 		t.Error("Didn't receive second Transcribed event.")
 	}
-	
+
 	err = <-transcriber.StopTranscribingAsync()
 	if err != nil {
 		t.Error("Got error: ", err)
 	}
-	
+
 	// Wait for session stopped event
 	select {
 	case <-sessionStoppedFuture:
 		t.Log("Received SessionStopped event.")
 	case <-time.After(30 * time.Second):
 		t.Error("Timeout waiting for SessionStopped event.")
 	}
-	
+
 	// Close the speaker ID channel to signal we're done collecting IDs
 	close(speakerIDsChan)
-	
+
 	// Collect unique speaker IDs
 	uniqueSpeakerIDs := make(map[string]bool)
 	for speakerID := range speakerIDsChan {
 		uniqueSpeakerIDs[speakerID] = true
 	}
-	
+
 	// Verify that more than one speaker ID was detected
 	if len(uniqueSpeakerIDs) <= 1 {
-		t.Errorf("Expected more than 1 unique speaker ID, but got %d: %v", 
+		t.Errorf("Expected more than 1 unique speaker ID, but got %d: %v",
 			len(uniqueSpeakerIDs), getKeysFromMap(uniqueSpeakerIDs))
 	} else {
-		t.Logf("Successfully detected %d unique speaker IDs: %v", 
+		t.Logf("Successfully detected %d unique speaker IDs: %v",
 			len(uniqueSpeakerIDs), getKeysFromMap(uniqueSpeakerIDs))
 	}
 }