From 6050317062c35b04f6f4809651bcd263e0d3742d Mon Sep 17 00:00:00 2001 From: SpencerPark Date: Wed, 13 Sep 2017 17:20:56 -0400 Subject: [PATCH 1/4] Create a test jupyter client for capturing interaction with the kernel during testing --- kernel_test.go | 367 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 259 insertions(+), 108 deletions(-) diff --git a/kernel_test.go b/kernel_test.go index 357ed19..f02c157 100644 --- a/kernel_test.go +++ b/kernel_test.go @@ -2,8 +2,9 @@ package main import ( "errors" + "fmt" "os" - "sync" + "strings" "testing" "time" @@ -15,6 +16,194 @@ const ( success = "\u2713" ) +const ( + connectionKey = "a0436f6c-1916-498b-8eb9-e81ab9368e84" + sessionID = "ba65a05c-106a-4799-9a94-7f5631bbe216" + transport = "tcp" + ip = "127.0.0.1" + shellPort = 57503 + iopubPort = 40885 +) + +type testJupyterClient struct { + shellSocket *zmq.Socket + ioSocket *zmq.Socket +} + +// newTestJupyterClient creates and connects a fresh client to the kernel. Upon error, newTestJupyterClient +// will Fail the test. +func newTestJupyterClient(t *testing.T) (testJupyterClient, func()) { + addrShell := fmt.Sprintf("%s://%s:%d", transport, ip, shellPort) + addrIO := fmt.Sprintf("%s://%s:%d", transport, ip, iopubPort) + + // Prepare the shell socket. + shell, err := zmq.NewSocket(zmq.REQ) + if err != nil { + t.Fatal("NewSocket:", err) + } + + if err = shell.Connect(addrShell); err != nil { + t.Fatal("shell.Connect:", err) + } + + // Prepare the IOPub socket. + iopub, err := zmq.NewSocket(zmq.SUB) + if err != nil { + t.Fatal("NewSocket:", err) + } + + if err = iopub.Connect(addrIO); err != nil { + t.Fatal("iopub.Connect:", err) + } + + if err = iopub.SetSubscribe(""); err != nil { + t.Fatal("iopub.SetSubscribe", err) + } + + // wait for a second to give the tcp connection time to complete to avoid missing the early pub messages + time.Sleep(1 * time.Second) + + return testJupyterClient{shell, iopub}, func() { + if err := shell.Close(); err != nil { + t.Fatal("shell.Close", err) + } + if err = iopub.Close(); err != nil { + t.Fatal("iopub.Close", err) + } + } +} + +// sendShellRequest sends a message to the kernel over the shell channel. Upon error, sendShellRequest +// will Fail the test. +func (client *testJupyterClient) sendShellRequest(t *testing.T, request ComposedMsg) { + if _, err := client.shellSocket.Send("", zmq.SNDMORE); err != nil { + t.Fatal("shellSocket.Send:", err) + } + + reqMsgParts, err := request.ToWireMsg([]byte(connectionKey)) + if err != nil { + t.Fatal("request.ToWireMsg:", err) + } + + if _, err = client.shellSocket.SendMessage(reqMsgParts); err != nil { + t.Fatal("shellSocket.SendMessage:", err) + } +} + +// recvShellReply tries to read a reply message from the shell channel. It will timeout after the given +// timeout delay. Upon error or timeout, recvShellReply will Fail the test. +func (client *testJupyterClient) recvShellReply(t *testing.T, timeout time.Duration) (reply ComposedMsg) { + ch := make(chan ComposedMsg) + + go func() { + repMsgParts, err := client.shellSocket.RecvMessageBytes(0) + if err != nil { + t.Fatal("Shell socket RecvMessageBytes:", err) + } + + msgParsed, _, err := WireMsgToComposedMsg(repMsgParts, []byte(connectionKey)) + if err != nil { + t.Fatal("Could not parse wire message:", err) + } + + ch <- msgParsed + }() + + select { + case reply = <-ch: + case <-time.After(timeout): + t.Fatal("recvShellReply timed out") + } + + return +} + +// recvIOSub tries to read a published message from the IOPub channel. It will timeout after the given +// timeout delay. Upon error or timeout, recvIOSub will Fail the test. +func (client *testJupyterClient) recvIOSub(t *testing.T, timeout time.Duration) (sub ComposedMsg) { + ch := make(chan ComposedMsg) + + go func() { + repMsgParts, err := client.ioSocket.RecvMessageBytes(0) + if err != nil { + t.Fatal("IOPub socket RecvMessageBytes:", err) + } + + msgParsed, _, err := WireMsgToComposedMsg(repMsgParts, []byte(connectionKey)) + if err != nil { + t.Fatal("Could not parse wire message:", err) + } + + ch <- msgParsed + }() + + select { + case sub = <-ch: + case <-time.After(timeout): + t.Fatal("recvIOSub timed out") + } + + return +} + +// request preforms a request and awaits a reply on the shell channel. Additionally all messages on the IOPub channel +// between the opening 'busy' messages and closing 'idle' message are captured and returned. The request will timeout +// after the given timeout delay. Upon error or timeout, request will Fail the test. +func (client *testJupyterClient) request(t *testing.T, request ComposedMsg, timeout time.Duration) (reply ComposedMsg, pub []ComposedMsg) { + client.sendShellRequest(t, request) + reply = client.recvShellReply(t, timeout) + + // Read the expected 'busy' message and ensure it is in fact, a 'busy' message + subMsg := client.recvIOSub(t, 1*time.Second) + if subMsg.Header.MsgType != "status" { + t.Fatalf("Expected a 'status' message but received a '%s' message on IOPub", subMsg.Header.MsgType) + } + + subData, ok := subMsg.Content.(map[string]interface{}) + if !ok { + t.Fatal("'status' message content is not a json object") + } + + execState, ok := subData["execution_state"] + if !ok { + t.Fatal("'status' message content is missing the 'execution_state' field") + } + + if execState != kernelBusy { + t.Fatalf("Expected a 'busy' status message but got '%v'", execState) + } + + // Read messages from the IOPub channel until an 'idle' message is received + for { + subMsg = client.recvIOSub(t, 100*time.Millisecond) + + // If the message is a 'status' message, ensure it is an 'idle' status + if subMsg.Header.MsgType == "status" { + subData, ok = subMsg.Content.(map[string]interface{}) + if !ok { + t.Fatal("'status' message content is not a json object") + } + + execState, ok = subData["execution_state"] + if !ok { + t.Fatal("'status' message content is missing the 'execution_state' field") + } + + if execState != kernelIdle { + t.Fatalf("Expected a 'idle' status message but got '%v'", execState) + } + + // Break from the loop as we don't expect any other IOPub messages after the 'idle' + break + } + + // Add the message to the pub collection + pub = append(pub, subMsg) + } + + return +} + func TestMain(m *testing.M) { os.Exit(runTest(m)) } @@ -34,13 +223,28 @@ func runTest(m *testing.M) int { // TestEvaluate tests the evaluation of consecutive cells.. func TestEvaluate(t *testing.T) { cases := []struct { - Input string + Input []string Output string }{ - {"import \"fmt\"\na := 1\nfmt.Println(a)", "1\n"}, - {"a = 2\nfmt.Println(a)", "2\n"}, - {"func myFunc(x int) int {\nreturn x+1\n}\nfmt.Println(\"func defined\")", "func dfined\n"}, - {"b := myFunc(1)\nfmt.Println(b)", "2\n"}, + {[]string{ + "import \"fmt\"", + "a := 1", + "fmt.Println(a)", + }, "1\n"}, + {[]string{ + "a = 2", + "fmt.Println(a)", + }, "2\n"}, + {[]string{ + "func myFunc(x int) int {", + " return x+1", + "}", + "fmt.Println(\"func defined\")", + }, "func defined\n"}, + {[]string{ + "b := myFunc(1)", + "fmt.Println(b)", + }, "2\n"}, } t.Logf("Should be able to evaluate valid code in notebook cells.") @@ -51,7 +255,7 @@ func TestEvaluate(t *testing.T) { t.Logf(" Evaluating code snippet %d/%d.", k+1, len(cases)) // Get the result. - result := testEvaluate(t, tc.Input, k) + result := testEvaluate(t, strings.Join(tc.Input, "\n")) // Compare the result. if result != tc.Output { @@ -63,136 +267,83 @@ func TestEvaluate(t *testing.T) { } // testEvaluate evaluates a cell. -func testEvaluate(t *testing.T, codeIn string, testCaseIndex int) string { - - // Define the shell socket. - addrShell := "tcp://127.0.0.1:57503" - addrIO := "tcp://127.0.0.1:40885" +func testEvaluate(t *testing.T, codeIn string) string { + client, closeClient := newTestJupyterClient(t) + defer closeClient() // Create a message. - msg, err := NewMsg("execute_request", ComposedMsg{}) + request, err := NewMsg("execute_request", ComposedMsg{}) if err != nil { - t.Fatal("Create New Message:", err) + t.Fatal("NewMessage:", err) } // Fill in remaining header information. - msg.Header.Session = "ba65a05c-106a-4799-9a94-7f5631bbe216" - msg.Header.Username = "blah" + request.Header.Session = sessionID + request.Header.Username = "KernelTester" // Fill in Metadata. - msg.Metadata = make(map[string]interface{}) + request.Metadata = make(map[string]interface{}) // Fill in content. content := make(map[string]interface{}) content["code"] = codeIn content["silent"] = false - msg.Content = content + request.Content = content - // Prepare the shell socket. - sock, err := zmq.NewSocket(zmq.REQ) - if err != nil { - t.Fatal("NewSocket:", err) + reply, pub := client.request(t, request, 10*time.Second) + + if reply.Header.MsgType != "execute_reply" { + t.Fatal("reply.Header.MsgType", errors.New("reply is not an 'execute_reply'")) } - defer sock.Close() - if err = sock.Connect(addrShell); err != nil { - t.Fatal("sock.Connect:", err) + content, ok := reply.Content.(map[string]interface{}) + if !ok { + t.Fatal("reply.Content.(map[string]interface{})", errors.New("reply content is not a json object")) } - // Prepare the IOPub subscriber. - sockIO, err := zmq.NewSocket(zmq.SUB) - if err != nil { - t.Fatal("NewSocket:", err) + statusRaw, ok := content["status"] + if !ok { + t.Fatal("content[\"status\"]", errors.New("status field not present in 'execute_reply'")) } - defer sockIO.Close() - if err = sockIO.Connect(addrIO); err != nil { - t.Fatal("sockIO.Connect:", err) + status, ok := statusRaw.(string) + if !ok { + t.Fatal("content[\"status\"]", errors.New("status field value is not a string")) } - sockIO.SetSubscribe("") + if status != "ok" { + t.Fatalf("Execution encountered error [%s]: %s", content["ename"], content["evalue"]) + } - // Start the subscriber. - quit := make(chan struct{}) - var result string - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - for { - select { - - case <-quit: - return - - default: - msgParts, err := sockIO.RecvMessageBytes(0) - if err != nil { - t.Fatal("sockIO.RecvMessageBytes:", err) - } - - msgParsed, _, err := WireMsgToComposedMsg(msgParts, []byte("a0436f6c-1916-498b-8eb9-e81ab9368e84")) - if err != nil { - t.Fatal("WireMsgToComposedMsg:", err) - } - - if msgParsed.Header.MsgType == "execute_result" { - content, ok := msgParsed.Content.(map[string]interface{}) - if !ok { - t.Fatal("msgParsed.Content.(map[string]interface{})", errors.New("Could not cast type")) - } - data, ok := content["data"] - if !ok { - t.Fatal("content[\"data\"]", errors.New("Data field not present")) - } - dataMap, ok := data.(map[string]interface{}) - if !ok { - t.Fatal("data.(map[string]string)", errors.New("Could not cast type")) - } - rawResult, ok := dataMap["text/plain"] - if !ok { - t.Fatal("dataMap[\"text/plain\"]", errors.New("text/plain field not present")) - } - result, ok = rawResult.(string) - if !ok { - t.Fatal("rawResult.(string)", errors.New("Could not cast result as string")) - } - return - } + for _, pubMsg := range pub { + if pubMsg.Header.MsgType == "execute_result" { + content, ok := pubMsg.Content.(map[string]interface{}) + if !ok { + t.Fatal("pubMsg.Content.(map[string]interface{})", errors.New("pubMsg 'execute_result' content is not a json object")) } - } - }() - time.Sleep(1 * time.Second) - - // Send the execute request. - if _, err := sock.Send("", zmq.SNDMORE); err != nil { - t.Fatal("sock.Send:", err) - } + bundledMIMEDataRaw, ok := content["data"] + if !ok { + t.Fatal("content[\"data\"]", errors.New("data field not present in 'execute_result'")) + } - msgParts, err := msg.ToWireMsg([]byte("a0436f6c-1916-498b-8eb9-e81ab9368e84")) - if err != nil { - t.Fatal("msg.ToWireMsg:", err) - } + bundledMIMEData, ok := bundledMIMEDataRaw.(map[string]interface{}) + if !ok { + t.Fatal("content[\"data\"]", errors.New("data field is not a MIME data bundle in 'execute_result'")) + } - if _, err = sock.SendMessage(msgParts); err != nil { - t.Fatal("sock.SendMessage:", err) - } + textRepRaw, ok := bundledMIMEData["text/plain"] + if !ok { + t.Fatal("content[\"data\"]", errors.New("data field doesn't contain a text representation in 'execute_result'")) + } - // Wait for the result. If we timeout, kill the subscriber. - done := make(chan struct{}) - go func() { - wg.Wait() - close(done) - }() + textRep, ok := textRepRaw.(string) + if !ok { + t.Fatal("content[\"data\"][\"text/plain\"]", errors.New("text representation is not a string in 'execute_result'")) + } - // Compare the result to the expect and clean up. - select { - case <-done: - return result - case <-time.After(10 * time.Second): - close(quit) - t.Fatalf("[test case %d] Evaution timed out!", testCaseIndex+1) + return textRep + } } return "" From 01f2b46e66e15a067b3dc8b40356dab746cdf8fa Mon Sep 17 00:00:00 2001 From: SpencerPark Date: Thu, 14 Sep 2017 13:26:46 -0400 Subject: [PATCH 2/4] Test that an uncaught panic produces the proper error messages --- kernel_test.go | 191 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 130 insertions(+), 61 deletions(-) diff --git a/kernel_test.go b/kernel_test.go index f02c157..20afa05 100644 --- a/kernel_test.go +++ b/kernel_test.go @@ -33,6 +33,8 @@ type testJupyterClient struct { // newTestJupyterClient creates and connects a fresh client to the kernel. Upon error, newTestJupyterClient // will Fail the test. func newTestJupyterClient(t *testing.T) (testJupyterClient, func()) { + t.Helper() + addrShell := fmt.Sprintf("%s://%s:%d", transport, ip, shellPort) addrIO := fmt.Sprintf("%s://%s:%d", transport, ip, iopubPort) @@ -76,6 +78,8 @@ func newTestJupyterClient(t *testing.T) (testJupyterClient, func()) { // sendShellRequest sends a message to the kernel over the shell channel. Upon error, sendShellRequest // will Fail the test. func (client *testJupyterClient) sendShellRequest(t *testing.T, request ComposedMsg) { + t.Helper() + if _, err := client.shellSocket.Send("", zmq.SNDMORE); err != nil { t.Fatal("shellSocket.Send:", err) } @@ -93,6 +97,8 @@ func (client *testJupyterClient) sendShellRequest(t *testing.T, request Composed // recvShellReply tries to read a reply message from the shell channel. It will timeout after the given // timeout delay. Upon error or timeout, recvShellReply will Fail the test. func (client *testJupyterClient) recvShellReply(t *testing.T, timeout time.Duration) (reply ComposedMsg) { + t.Helper() + ch := make(chan ComposedMsg) go func() { @@ -121,6 +127,8 @@ func (client *testJupyterClient) recvShellReply(t *testing.T, timeout time.Durat // recvIOSub tries to read a published message from the IOPub channel. It will timeout after the given // timeout delay. Upon error or timeout, recvIOSub will Fail the test. func (client *testJupyterClient) recvIOSub(t *testing.T, timeout time.Duration) (sub ComposedMsg) { + t.Helper() + ch := make(chan ComposedMsg) go func() { @@ -150,24 +158,17 @@ func (client *testJupyterClient) recvIOSub(t *testing.T, timeout time.Duration) // between the opening 'busy' messages and closing 'idle' message are captured and returned. The request will timeout // after the given timeout delay. Upon error or timeout, request will Fail the test. func (client *testJupyterClient) request(t *testing.T, request ComposedMsg, timeout time.Duration) (reply ComposedMsg, pub []ComposedMsg) { + t.Helper() + client.sendShellRequest(t, request) reply = client.recvShellReply(t, timeout) // Read the expected 'busy' message and ensure it is in fact, a 'busy' message subMsg := client.recvIOSub(t, 1*time.Second) - if subMsg.Header.MsgType != "status" { - t.Fatalf("Expected a 'status' message but received a '%s' message on IOPub", subMsg.Header.MsgType) - } + assertMsgTypeEquals(t, subMsg, "status") - subData, ok := subMsg.Content.(map[string]interface{}) - if !ok { - t.Fatal("'status' message content is not a json object") - } - - execState, ok := subData["execution_state"] - if !ok { - t.Fatal("'status' message content is missing the 'execution_state' field") - } + subData := getMsgContentAsJsonObject(t, subMsg) + execState := getString(t, "content", subData, "execution_state") if execState != kernelBusy { t.Fatalf("Expected a 'busy' status message but got '%v'", execState) @@ -179,15 +180,8 @@ func (client *testJupyterClient) request(t *testing.T, request ComposedMsg, time // If the message is a 'status' message, ensure it is an 'idle' status if subMsg.Header.MsgType == "status" { - subData, ok = subMsg.Content.(map[string]interface{}) - if !ok { - t.Fatal("'status' message content is not a json object") - } - - execState, ok = subData["execution_state"] - if !ok { - t.Fatal("'status' message content is missing the 'execution_state' field") - } + subData = getMsgContentAsJsonObject(t, subMsg) + execState = getString(t, "content", subData, "execution_state") if execState != kernelIdle { t.Fatalf("Expected a 'idle' status message but got '%v'", execState) @@ -204,6 +198,67 @@ func (client *testJupyterClient) request(t *testing.T, request ComposedMsg, time return } +// assertMsgTypeEquals is a test helper that fails the test if the message header's MsgType is not the +// expectedType. +func assertMsgTypeEquals(t *testing.T, msg ComposedMsg, expectedType string) { + t.Helper() + + if msg.Header.MsgType != expectedType { + t.Fatalf("Expected message of type '%s' but was '%s'\n", expectedType, msg.Header.MsgType) + } +} + +// getMsgContentAsJsonObject is a test helper that fails the rest if the message content is not a +// map[string]interface{} and returns the content as a map[string]interface{} if it is of the correct type. +func getMsgContentAsJsonObject(t *testing.T, msg ComposedMsg) map[string]interface{} { + t.Helper() + + content, ok := msg.Content.(map[string]interface{}) + if !ok { + t.Fatal("Message content is not a JSON object") + } + + return content +} + +// getString is a test helper that retrieves a value as a string from the content at the given key. If the key +// does not exist in the content map or the value is not a string this will fail the test. The jsonObjectName +// parameter is a string used to name the content for more helpful fail messages. +func getString(t *testing.T, jsonObjectName string, content map[string]interface{}, key string) string { + t.Helper() + + raw, ok := content[key] + if !ok { + t.Fatal(jsonObjectName+"[\""+key+"\"]", errors.New("\""+key+"\" field not present")) + } + + value, ok := raw.(string) + if !ok { + t.Fatal(jsonObjectName+"[\""+key+"\"]", errors.New("\""+key+"\" is not a string")) + } + + return value +} + +// getString is a test helper that retrieves a value as a map[string]interface{} from the content at the given key. +// If the key does not exist in the content map or the value is not a map[string]interface{} this will fail the test. +// The jsonObjectName parameter is a string used to name the content for more helpful fail messages. +func getJsonObject(t *testing.T, jsonObjectName string, content map[string]interface{}, key string) map[string]interface{} { + t.Helper() + + raw, ok := content[key] + if !ok { + t.Fatal(jsonObjectName+"[\""+key+"\"]", errors.New("\""+key+"\" field not present")) + } + + value, ok := raw.(map[string]interface{}) + if !ok { + t.Fatal(jsonObjectName+"[\""+key+"\"]", errors.New("\""+key+"\" is not a string")) + } + + return value +} + func TestMain(m *testing.M) { os.Exit(runTest(m)) } @@ -220,7 +275,7 @@ func runTest(m *testing.M) int { //============================================================================== -// TestEvaluate tests the evaluation of consecutive cells.. +// TestEvaluate tests the evaluation of consecutive cells. func TestEvaluate(t *testing.T) { cases := []struct { Input []string @@ -292,24 +347,10 @@ func testEvaluate(t *testing.T, codeIn string) string { reply, pub := client.request(t, request, 10*time.Second) - if reply.Header.MsgType != "execute_reply" { - t.Fatal("reply.Header.MsgType", errors.New("reply is not an 'execute_reply'")) - } + assertMsgTypeEquals(t, reply, "execute_reply") - content, ok := reply.Content.(map[string]interface{}) - if !ok { - t.Fatal("reply.Content.(map[string]interface{})", errors.New("reply content is not a json object")) - } - - statusRaw, ok := content["status"] - if !ok { - t.Fatal("content[\"status\"]", errors.New("status field not present in 'execute_reply'")) - } - - status, ok := statusRaw.(string) - if !ok { - t.Fatal("content[\"status\"]", errors.New("status field value is not a string")) - } + content = getMsgContentAsJsonObject(t, reply) + status := getString(t, "content", content, "status") if status != "ok" { t.Fatalf("Execution encountered error [%s]: %s", content["ename"], content["evalue"]) @@ -317,34 +358,62 @@ func testEvaluate(t *testing.T, codeIn string) string { for _, pubMsg := range pub { if pubMsg.Header.MsgType == "execute_result" { - content, ok := pubMsg.Content.(map[string]interface{}) - if !ok { - t.Fatal("pubMsg.Content.(map[string]interface{})", errors.New("pubMsg 'execute_result' content is not a json object")) - } + content = getMsgContentAsJsonObject(t, pubMsg) - bundledMIMEDataRaw, ok := content["data"] - if !ok { - t.Fatal("content[\"data\"]", errors.New("data field not present in 'execute_result'")) - } + bundledMIMEData := getJsonObject(t, "content", content, "data") + textRep := getString(t, "content[\"data\"]", bundledMIMEData, "test/plain") - bundledMIMEData, ok := bundledMIMEDataRaw.(map[string]interface{}) - if !ok { - t.Fatal("content[\"data\"]", errors.New("data field is not a MIME data bundle in 'execute_result'")) - } + return textRep + } + } - textRepRaw, ok := bundledMIMEData["text/plain"] - if !ok { - t.Fatal("content[\"data\"]", errors.New("data field doesn't contain a text representation in 'execute_result'")) - } + return "" +} - textRep, ok := textRepRaw.(string) - if !ok { - t.Fatal("content[\"data\"][\"text/plain\"]", errors.New("text representation is not a string in 'execute_result'")) - } +// TestPanicGeneratesError tests that executing code with an un-recovered panic properly generates both +// an error "execute_reply" and publishes an "error" message. +func TestPanicGeneratesError(t *testing.T) { + client, closeClient := newTestJupyterClient(t) + defer closeClient() - return textRep + // Create a message. + request, err := NewMsg("execute_request", ComposedMsg{}) + if err != nil { + t.Fatal("NewMessage:", err) + } + + // Fill in remaining header information. + request.Header.Session = sessionID + request.Header.Username = "KernelTester" + + // Fill in Metadata. + request.Metadata = make(map[string]interface{}) + + // Fill in content. + content := make(map[string]interface{}) + content["code"] = "panic(\"Error\")" + content["silent"] = false + request.Content = content + + reply, pub := client.request(t, request, 10*time.Second) + + assertMsgTypeEquals(t, reply, "execute_reply") + + content = getMsgContentAsJsonObject(t, reply) + status := getString(t, "content", content, "status") + + if status != "error" { + t.Fatal("Execution did not raise expected error") + } + + foundPublishedError := false + for _, pubMsg := range pub { + if pubMsg.Header.MsgType == "error" { + foundPublishedError = true } } - return "" + if !foundPublishedError { + t.Fatal("Execution did not publish an expected \"error\" message") + } } From 922e21eac862cb6736d09d36440ba8f16ecb6f2b Mon Sep 17 00:00:00 2001 From: SpencerPark Date: Fri, 15 Sep 2017 15:57:36 -0400 Subject: [PATCH 3/4] Reformat kernel_test. - Normalize fatal messages to be prefixed with a tab char and error symbol. - Move test helpers to the bottom and better seperate them from the tests. - Read kernel connection info from the test fixture file. --- kernel_test.go | 431 ++++++++++++++++++++++++++----------------------- 1 file changed, 230 insertions(+), 201 deletions(-) diff --git a/kernel_test.go b/kernel_test.go index 20afa05..b870927 100644 --- a/kernel_test.go +++ b/kernel_test.go @@ -1,8 +1,10 @@ package main import ( - "errors" + "encoding/json" "fmt" + "io/ioutil" + "log" "os" "strings" "testing" @@ -16,15 +18,201 @@ const ( success = "\u2713" ) -const ( - connectionKey = "a0436f6c-1916-498b-8eb9-e81ab9368e84" - sessionID = "ba65a05c-106a-4799-9a94-7f5631bbe216" - transport = "tcp" - ip = "127.0.0.1" - shellPort = 57503 - iopubPort = 40885 +const sessionID = "ba65a05c-106a-4799-9a94-7f5631bbe216" + +var ( + connectionKey string + transport string + ip string + shellPort int + iopubPort int ) +//============================================================================== + +func TestMain(m *testing.M) { + os.Exit(runTest(m)) +} + +// runTest initializes the environment for the tests and allows for +// the proper exit if the test fails or succeeds. +func runTest(m *testing.M) int { + const connectionFile = "fixtures/connection_file.json" + + // Parse the connection info. + var connInfo ConnectionInfo + + connData, err := ioutil.ReadFile(connectionFile) + if err != nil { + log.Fatal(err) + } + + if err = json.Unmarshal(connData, &connInfo); err != nil { + log.Fatal(err) + } + + // Store the connection parameters globally for use by the test client. + connectionKey = connInfo.Key + transport = connInfo.Transport + ip = connInfo.IP + shellPort = connInfo.ShellPort + iopubPort = connInfo.IOPubPort + + // Start the kernel. + go runKernel(connectionFile) + + return m.Run() +} + +//============================================================================== + +// TestEvaluate tests the evaluation of consecutive cells. +func TestEvaluate(t *testing.T) { + cases := []struct { + Input []string + Output string + }{ + {[]string{ + "import \"fmt\"", + "a := 1", + "fmt.Println(a)", + }, "1\n"}, + {[]string{ + "a = 2", + "fmt.Println(a)", + }, "2\n"}, + {[]string{ + "func myFunc(x int) int {", + " return x+1", + "}", + "fmt.Println(\"func defined\")", + }, "func defined\n"}, + {[]string{ + "b := myFunc(1)", + "fmt.Println(b)", + }, "2\n"}, + } + + t.Logf("Should be able to evaluate valid code in notebook cells.") + + for k, tc := range cases { + + // Give a progress report. + t.Logf(" Evaluating code snippet %d/%d.", k+1, len(cases)) + + // Get the result. + result := testEvaluate(t, strings.Join(tc.Input, "\n")) + + // Compare the result. + if result != tc.Output { + t.Errorf("\t%s Test case produced unexpected results.", failure) + continue + } + t.Logf("\t%s Should return the correct cell output.", success) + } +} + +// testEvaluate evaluates a cell. +func testEvaluate(t *testing.T, codeIn string) string { + client, closeClient := newTestJupyterClient(t) + defer closeClient() + + // Create a message. + request, err := NewMsg("execute_request", ComposedMsg{}) + if err != nil { + t.Fatalf("\t%s NewMsg: %s", failure, err) + } + + // Fill in remaining header information. + request.Header.Session = sessionID + request.Header.Username = "KernelTester" + + // Fill in Metadata. + request.Metadata = make(map[string]interface{}) + + // Fill in content. + content := make(map[string]interface{}) + content["code"] = codeIn + content["silent"] = false + request.Content = content + + reply, pub := client.performJupyterRequest(t, request, 10*time.Second) + + assertMsgTypeEquals(t, reply, "execute_reply") + + content = getMsgContentAsJsonObject(t, reply) + status := getString(t, "content", content, "status") + + if status != "ok" { + t.Fatalf("\t%s Execution encountered error [%s]: %s", failure, content["ename"], content["evalue"]) + } + + for _, pubMsg := range pub { + if pubMsg.Header.MsgType == "execute_result" { + content = getMsgContentAsJsonObject(t, pubMsg) + + bundledMIMEData := getJsonObject(t, "content", content, "data") + textRep := getString(t, "content[\"data\"]", bundledMIMEData, "text/plain") + + return textRep + } + } + + return "" +} + +// TestPanicGeneratesError tests that executing code with an un-recovered panic properly generates both +// an error "execute_reply" and publishes an "error" message. +func TestPanicGeneratesError(t *testing.T) { + client, closeClient := newTestJupyterClient(t) + defer closeClient() + + // Create a message. + request, err := NewMsg("execute_request", ComposedMsg{}) + if err != nil { + t.Fatalf("\t%s NewMsg: %s", failure, err) + } + + // Fill in remaining header information. + request.Header.Session = sessionID + request.Header.Username = "KernelTester" + + // Fill in Metadata. + request.Metadata = make(map[string]interface{}) + + // Fill in content. + content := make(map[string]interface{}) + content["code"] = "panic(\"Error\")" + content["silent"] = false + request.Content = content + + reply, pub := client.performJupyterRequest(t, request, 10*time.Second) + + assertMsgTypeEquals(t, reply, "execute_reply") + + content = getMsgContentAsJsonObject(t, reply) + status := getString(t, "content", content, "status") + + if status != "error" { + t.Fatalf("\t%s Execution did not raise expected error", failure) + } + + var foundPublishedError bool + for _, pubMsg := range pub { + if pubMsg.Header.MsgType == "error" { + foundPublishedError = true + break + } + } + + if !foundPublishedError { + t.Fatalf("\t%s Execution did not publish an expected \"error\" message", failure) + } +} + +//============================================================================== + +// testJupyterClient holds references to the 2 sockets it uses to communicate with the kernel. type testJupyterClient struct { shellSocket *zmq.Socket ioSocket *zmq.Socket @@ -41,36 +229,36 @@ func newTestJupyterClient(t *testing.T) (testJupyterClient, func()) { // Prepare the shell socket. shell, err := zmq.NewSocket(zmq.REQ) if err != nil { - t.Fatal("NewSocket:", err) + t.Fatalf("\t%s NewSocket: %s", failure, err) } if err = shell.Connect(addrShell); err != nil { - t.Fatal("shell.Connect:", err) + t.Fatalf("\t%s shell.Connect: %s", failure, err) } // Prepare the IOPub socket. iopub, err := zmq.NewSocket(zmq.SUB) if err != nil { - t.Fatal("NewSocket:", err) + t.Fatalf("\t%s NewSocket: %s", failure, err) } if err = iopub.Connect(addrIO); err != nil { - t.Fatal("iopub.Connect:", err) + t.Fatalf("\t%s iopub.Connect: %s", failure, err) } if err = iopub.SetSubscribe(""); err != nil { - t.Fatal("iopub.SetSubscribe", err) + t.Fatalf("\t%s iopub.SetSubscribe: %s", failure, err) } - // wait for a second to give the tcp connection time to complete to avoid missing the early pub messages + // Wait for a second to give the tcp connection time to complete to avoid missing the early pub messages. time.Sleep(1 * time.Second) return testJupyterClient{shell, iopub}, func() { if err := shell.Close(); err != nil { - t.Fatal("shell.Close", err) + t.Errorf("\t%s shell.Close: %s", failure, err) } if err = iopub.Close(); err != nil { - t.Fatal("iopub.Close", err) + t.Errorf("\t%s iopub.Close: %s", failure, err) } } } @@ -81,16 +269,16 @@ func (client *testJupyterClient) sendShellRequest(t *testing.T, request Composed t.Helper() if _, err := client.shellSocket.Send("", zmq.SNDMORE); err != nil { - t.Fatal("shellSocket.Send:", err) + t.Fatalf("\t%s shellSocket.Send: %s", failure, err) } reqMsgParts, err := request.ToWireMsg([]byte(connectionKey)) if err != nil { - t.Fatal("request.ToWireMsg:", err) + t.Fatalf("\t%s request.ToWireMsg: %s", failure, err) } if _, err = client.shellSocket.SendMessage(reqMsgParts); err != nil { - t.Fatal("shellSocket.SendMessage:", err) + t.Fatalf("\t%s shellSocket.SendMessage: %s", failure, err) } } @@ -104,12 +292,12 @@ func (client *testJupyterClient) recvShellReply(t *testing.T, timeout time.Durat go func() { repMsgParts, err := client.shellSocket.RecvMessageBytes(0) if err != nil { - t.Fatal("Shell socket RecvMessageBytes:", err) + t.Fatalf("\t%s Shell socket RecvMessageBytes: %s", failure, err) } msgParsed, _, err := WireMsgToComposedMsg(repMsgParts, []byte(connectionKey)) if err != nil { - t.Fatal("Could not parse wire message:", err) + t.Fatalf("\t%s Could not parse wire message: %s", failure, err) } ch <- msgParsed @@ -118,7 +306,7 @@ func (client *testJupyterClient) recvShellReply(t *testing.T, timeout time.Durat select { case reply = <-ch: case <-time.After(timeout): - t.Fatal("recvShellReply timed out") + t.Fatalf("\t%s recvShellReply timed out", failure) } return @@ -134,12 +322,12 @@ func (client *testJupyterClient) recvIOSub(t *testing.T, timeout time.Duration) go func() { repMsgParts, err := client.ioSocket.RecvMessageBytes(0) if err != nil { - t.Fatal("IOPub socket RecvMessageBytes:", err) + t.Fatalf("\t%s IOPub socket RecvMessageBytes: %s", failure, err) } msgParsed, _, err := WireMsgToComposedMsg(repMsgParts, []byte(connectionKey)) if err != nil { - t.Fatal("Could not parse wire message:", err) + t.Fatalf("\t%s Could not parse wire message: %s", failure, err) } ch <- msgParsed @@ -148,22 +336,22 @@ func (client *testJupyterClient) recvIOSub(t *testing.T, timeout time.Duration) select { case sub = <-ch: case <-time.After(timeout): - t.Fatal("recvIOSub timed out") + t.Fatalf("\t%s recvIOSub timed out", failure) } return } -// request preforms a request and awaits a reply on the shell channel. Additionally all messages on the IOPub channel -// between the opening 'busy' messages and closing 'idle' message are captured and returned. The request will timeout -// after the given timeout delay. Upon error or timeout, request will Fail the test. -func (client *testJupyterClient) request(t *testing.T, request ComposedMsg, timeout time.Duration) (reply ComposedMsg, pub []ComposedMsg) { +// performJupyterRequest preforms a request and awaits a reply on the shell channel. Additionally all messages on the +// IOPub channel between the opening 'busy' messages and closing 'idle' message are captured and returned. The request +// will timeout after the given timeout delay. Upon error or timeout, request will Fail the test. +func (client *testJupyterClient) performJupyterRequest(t *testing.T, request ComposedMsg, timeout time.Duration) (reply ComposedMsg, pub []ComposedMsg) { t.Helper() client.sendShellRequest(t, request) reply = client.recvShellReply(t, timeout) - // Read the expected 'busy' message and ensure it is in fact, a 'busy' message + // Read the expected 'busy' message and ensure it is in fact, a 'busy' message. subMsg := client.recvIOSub(t, 1*time.Second) assertMsgTypeEquals(t, subMsg, "status") @@ -171,27 +359,27 @@ func (client *testJupyterClient) request(t *testing.T, request ComposedMsg, time execState := getString(t, "content", subData, "execution_state") if execState != kernelBusy { - t.Fatalf("Expected a 'busy' status message but got '%v'", execState) + t.Fatalf("\t%s Expected a 'busy' status message but got '%s'", failure, execState) } - // Read messages from the IOPub channel until an 'idle' message is received + // Read messages from the IOPub channel until an 'idle' message is received. for { subMsg = client.recvIOSub(t, 100*time.Millisecond) - // If the message is a 'status' message, ensure it is an 'idle' status + // If the message is a 'status' message, ensure it is an 'idle' status. if subMsg.Header.MsgType == "status" { subData = getMsgContentAsJsonObject(t, subMsg) execState = getString(t, "content", subData, "execution_state") if execState != kernelIdle { - t.Fatalf("Expected a 'idle' status message but got '%v'", execState) + t.Fatalf("\t%s Expected a 'idle' status message but got '%s'", failure, execState) } - // Break from the loop as we don't expect any other IOPub messages after the 'idle' + // Break from the loop as we don't expect any other IOPub messages after the 'idle'. break } - // Add the message to the pub collection + // Add the message to the pub collection. pub = append(pub, subMsg) } @@ -204,7 +392,7 @@ func assertMsgTypeEquals(t *testing.T, msg ComposedMsg, expectedType string) { t.Helper() if msg.Header.MsgType != expectedType { - t.Fatalf("Expected message of type '%s' but was '%s'\n", expectedType, msg.Header.MsgType) + t.Fatalf("\t%s Expected message of type '%s' but was '%s'", failure, expectedType, msg.Header.MsgType) } } @@ -215,7 +403,7 @@ func getMsgContentAsJsonObject(t *testing.T, msg ComposedMsg) map[string]interfa content, ok := msg.Content.(map[string]interface{}) if !ok { - t.Fatal("Message content is not a JSON object") + t.Fatalf("\t%s Message content is not a JSON object", failure) } return content @@ -229,12 +417,12 @@ func getString(t *testing.T, jsonObjectName string, content map[string]interface raw, ok := content[key] if !ok { - t.Fatal(jsonObjectName+"[\""+key+"\"]", errors.New("\""+key+"\" field not present")) + t.Fatalf("\t%s %s[\"%s\"] field not present", failure, jsonObjectName, key) } value, ok := raw.(string) if !ok { - t.Fatal(jsonObjectName+"[\""+key+"\"]", errors.New("\""+key+"\" is not a string")) + t.Fatalf("\t%s %s[\"%s\"] is not a string", failure, jsonObjectName, key) } return value @@ -248,172 +436,13 @@ func getJsonObject(t *testing.T, jsonObjectName string, content map[string]inter raw, ok := content[key] if !ok { - t.Fatal(jsonObjectName+"[\""+key+"\"]", errors.New("\""+key+"\" field not present")) + t.Fatalf("\t%s %s[\"%s\"] field not present", failure, jsonObjectName, key) } value, ok := raw.(map[string]interface{}) if !ok { - t.Fatal(jsonObjectName+"[\""+key+"\"]", errors.New("\""+key+"\" is not a string")) + t.Fatalf("\t%s %s[\"%s\"] is not a JSON object", failure, jsonObjectName, key) } return value } - -func TestMain(m *testing.M) { - os.Exit(runTest(m)) -} - -// runTest initializes the environment for the tests and allows for -// the proper exit if the test fails or succeeds. -func runTest(m *testing.M) int { - - // Start the kernel. - go runKernel("fixtures/connection_file.json") - - return m.Run() -} - -//============================================================================== - -// TestEvaluate tests the evaluation of consecutive cells. -func TestEvaluate(t *testing.T) { - cases := []struct { - Input []string - Output string - }{ - {[]string{ - "import \"fmt\"", - "a := 1", - "fmt.Println(a)", - }, "1\n"}, - {[]string{ - "a = 2", - "fmt.Println(a)", - }, "2\n"}, - {[]string{ - "func myFunc(x int) int {", - " return x+1", - "}", - "fmt.Println(\"func defined\")", - }, "func defined\n"}, - {[]string{ - "b := myFunc(1)", - "fmt.Println(b)", - }, "2\n"}, - } - - t.Logf("Should be able to evaluate valid code in notebook cells.") - - for k, tc := range cases { - - // Give a progress report. - t.Logf(" Evaluating code snippet %d/%d.", k+1, len(cases)) - - // Get the result. - result := testEvaluate(t, strings.Join(tc.Input, "\n")) - - // Compare the result. - if result != tc.Output { - t.Errorf("\t%s Test case produced unexpected results.", failure) - continue - } - t.Logf("\t%s Should return the correct cell output.", success) - } -} - -// testEvaluate evaluates a cell. -func testEvaluate(t *testing.T, codeIn string) string { - client, closeClient := newTestJupyterClient(t) - defer closeClient() - - // Create a message. - request, err := NewMsg("execute_request", ComposedMsg{}) - if err != nil { - t.Fatal("NewMessage:", err) - } - - // Fill in remaining header information. - request.Header.Session = sessionID - request.Header.Username = "KernelTester" - - // Fill in Metadata. - request.Metadata = make(map[string]interface{}) - - // Fill in content. - content := make(map[string]interface{}) - content["code"] = codeIn - content["silent"] = false - request.Content = content - - reply, pub := client.request(t, request, 10*time.Second) - - assertMsgTypeEquals(t, reply, "execute_reply") - - content = getMsgContentAsJsonObject(t, reply) - status := getString(t, "content", content, "status") - - if status != "ok" { - t.Fatalf("Execution encountered error [%s]: %s", content["ename"], content["evalue"]) - } - - for _, pubMsg := range pub { - if pubMsg.Header.MsgType == "execute_result" { - content = getMsgContentAsJsonObject(t, pubMsg) - - bundledMIMEData := getJsonObject(t, "content", content, "data") - textRep := getString(t, "content[\"data\"]", bundledMIMEData, "test/plain") - - return textRep - } - } - - return "" -} - -// TestPanicGeneratesError tests that executing code with an un-recovered panic properly generates both -// an error "execute_reply" and publishes an "error" message. -func TestPanicGeneratesError(t *testing.T) { - client, closeClient := newTestJupyterClient(t) - defer closeClient() - - // Create a message. - request, err := NewMsg("execute_request", ComposedMsg{}) - if err != nil { - t.Fatal("NewMessage:", err) - } - - // Fill in remaining header information. - request.Header.Session = sessionID - request.Header.Username = "KernelTester" - - // Fill in Metadata. - request.Metadata = make(map[string]interface{}) - - // Fill in content. - content := make(map[string]interface{}) - content["code"] = "panic(\"Error\")" - content["silent"] = false - request.Content = content - - reply, pub := client.request(t, request, 10*time.Second) - - assertMsgTypeEquals(t, reply, "execute_reply") - - content = getMsgContentAsJsonObject(t, reply) - status := getString(t, "content", content, "status") - - if status != "error" { - t.Fatal("Execution did not raise expected error") - } - - foundPublishedError := false - for _, pubMsg := range pub { - if pubMsg.Header.MsgType == "error" { - foundPublishedError = true - } - } - - if !foundPublishedError { - t.Fatal("Execution did not publish an expected \"error\" message") - } -} From aee1cad4690ebb6b6b5621d5f71b9f7392340a4e Mon Sep 17 00:00:00 2001 From: SpencerPark Date: Mon, 18 Sep 2017 12:41:23 -0400 Subject: [PATCH 4/4] Fix kernel_test lint issues --- kernel_test.go | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/kernel_test.go b/kernel_test.go index b870927..c772158 100644 --- a/kernel_test.go +++ b/kernel_test.go @@ -18,7 +18,10 @@ const ( success = "\u2713" ) -const sessionID = "ba65a05c-106a-4799-9a94-7f5631bbe216" +const ( + connectionFile = "fixtures/connection_file.json" + sessionID = "ba65a05c-106a-4799-9a94-7f5631bbe216" +) var ( connectionKey string @@ -37,8 +40,6 @@ func TestMain(m *testing.M) { // runTest initializes the environment for the tests and allows for // the proper exit if the test fails or succeeds. func runTest(m *testing.M) int { - const connectionFile = "fixtures/connection_file.json" - // Parse the connection info. var connInfo ConnectionInfo @@ -140,7 +141,7 @@ func testEvaluate(t *testing.T, codeIn string) string { assertMsgTypeEquals(t, reply, "execute_reply") - content = getMsgContentAsJsonObject(t, reply) + content = getMsgContentAsJSONObject(t, reply) status := getString(t, "content", content, "status") if status != "ok" { @@ -149,9 +150,9 @@ func testEvaluate(t *testing.T, codeIn string) string { for _, pubMsg := range pub { if pubMsg.Header.MsgType == "execute_result" { - content = getMsgContentAsJsonObject(t, pubMsg) + content = getMsgContentAsJSONObject(t, pubMsg) - bundledMIMEData := getJsonObject(t, "content", content, "data") + bundledMIMEData := getJSONObject(t, "content", content, "data") textRep := getString(t, "content[\"data\"]", bundledMIMEData, "text/plain") return textRep @@ -190,7 +191,7 @@ func TestPanicGeneratesError(t *testing.T) { assertMsgTypeEquals(t, reply, "execute_reply") - content = getMsgContentAsJsonObject(t, reply) + content = getMsgContentAsJSONObject(t, reply) status := getString(t, "content", content, "status") if status != "error" { @@ -355,7 +356,7 @@ func (client *testJupyterClient) performJupyterRequest(t *testing.T, request Com subMsg := client.recvIOSub(t, 1*time.Second) assertMsgTypeEquals(t, subMsg, "status") - subData := getMsgContentAsJsonObject(t, subMsg) + subData := getMsgContentAsJSONObject(t, subMsg) execState := getString(t, "content", subData, "execution_state") if execState != kernelBusy { @@ -368,7 +369,7 @@ func (client *testJupyterClient) performJupyterRequest(t *testing.T, request Com // If the message is a 'status' message, ensure it is an 'idle' status. if subMsg.Header.MsgType == "status" { - subData = getMsgContentAsJsonObject(t, subMsg) + subData = getMsgContentAsJSONObject(t, subMsg) execState = getString(t, "content", subData, "execution_state") if execState != kernelIdle { @@ -396,9 +397,9 @@ func assertMsgTypeEquals(t *testing.T, msg ComposedMsg, expectedType string) { } } -// getMsgContentAsJsonObject is a test helper that fails the rest if the message content is not a +// getMsgContentAsJSONObject is a test helper that fails the rest if the message content is not a // map[string]interface{} and returns the content as a map[string]interface{} if it is of the correct type. -func getMsgContentAsJsonObject(t *testing.T, msg ComposedMsg) map[string]interface{} { +func getMsgContentAsJSONObject(t *testing.T, msg ComposedMsg) map[string]interface{} { t.Helper() content, ok := msg.Content.(map[string]interface{}) @@ -428,10 +429,10 @@ func getString(t *testing.T, jsonObjectName string, content map[string]interface return value } -// getString is a test helper that retrieves a value as a map[string]interface{} from the content at the given key. +// getJSONObject is a test helper that retrieves a value as a map[string]interface{} from the content at the given key. // If the key does not exist in the content map or the value is not a map[string]interface{} this will fail the test. // The jsonObjectName parameter is a string used to name the content for more helpful fail messages. -func getJsonObject(t *testing.T, jsonObjectName string, content map[string]interface{}, key string) map[string]interface{} { +func getJSONObject(t *testing.T, jsonObjectName string, content map[string]interface{}, key string) map[string]interface{} { t.Helper() raw, ok := content[key]