Commit 6cb23744 authored by Jonathan Wilkes's avatar Jonathan Wilkes

add regression tests for changes to message box (and future regressions)

This is an initial attempt at a regression testing framework.

Initially it has some regression tests that come from the work on
improvements to the msg box.

The framework allows the tests themselves to remain extremely simple. They
just need to send a number and potential error message to the outlet.

However, they do need to be added in the main patch. This is because we
will need to run some tests in a certain order. We could number the test
filenames but that's less easy to change later.
parent ff8afc1c
Pipeline #1270 passed with stage
in 333 minutes and 50 seconds
......@@ -11,6 +11,7 @@ debian_stable_i386_deb:
- ./tar_em_up.sh -B
- cd ..
- echo "" | sudo -S DEBIAN_FRONTEND=noninteractive dpkg --force-all -i pd-l2ork*.deb > /dev/null
- valgrind pd-l2ork -noprefs -nostdpath -nogui -nrt -noaudio -send "init dollarzero \$0" scripts/regression_tests.pd
- valgrind pd-l2ork -noprefs -nostdpath -nogui -nrt -noaudio scripts/external-tests.pd
artifacts:
name: "$CI_RUNNER_DESCRIPTION"
......@@ -31,6 +32,7 @@ debian_stable_amd64_deb:
- ./tar_em_up.sh -B
- cd ..
- echo "" | sudo -S DEBIAN_FRONTEND=noninteractive dpkg --force-all -i pd-l2ork*.deb > /dev/null
- valgrind pd-l2ork -noprefs -nostdpath -nogui -nrt -noaudio -send "init dollarzero \$0" scripts/regression_tests.pd
- valgrind pd-l2ork -noprefs -nostdpath -nogui -nrt -noaudio scripts/external-tests.pd
artifacts:
name: "$CI_RUNNER_DESCRIPTION"
......@@ -51,6 +53,7 @@ debian_stretch_amd64_deb:
- ./tar_em_up.sh -B
- cd ..
- echo "" | sudo -S DEBIAN_FRONTEND=noninteractive dpkg --force-all -i pd-l2ork*.deb > /dev/null
- valgrind pd-l2ork -noprefs -nostdpath -nogui -nrt -noaudio -send "init dollarzero \$0" scripts/regression_tests.pd
- valgrind pd-l2ork -noprefs -nostdpath -nogui -nrt -noaudio scripts/external-tests.pd
artifacts:
name: "$CI_RUNNER_DESCRIPTION"
......@@ -72,6 +75,7 @@ ubuntu_14.04_i386_deb:
- cd ..
- echo "" | sudo -S DEBIAN_FRONTEND=noninteractive dpkg --force-all -i pd-l2ork*.deb > /dev/null
- cd scripts
- valgrind pd-l2ork -noprefs -nostdpath -nogui -nrt -noaudio -send "init dollarzero \$0" regression_tests.pd
- valgrind pd-l2ork -noprefs -nostdpath -nogui -nrt -noaudio external-tests.pd
artifacts:
name: "$CI_RUNNER_DESCRIPTION"
......@@ -93,6 +97,7 @@ ubuntu_14.04_amd64_deb:
- cd ..
- echo "" | sudo -S DEBIAN_FRONTEND=noninteractive dpkg --force-all -i pd-l2ork*.deb > /dev/null
- cd scripts
- valgrind pd-l2ork -noprefs -nostdpath -nogui -nrt -noaudio -send "init dollarzero \$0" regression_tests.pd
- valgrind pd-l2ork -noprefs -nostdpath -nogui -nrt -noaudio external-tests.pd
artifacts:
name: "$CI_RUNNER_DESCRIPTION"
......@@ -114,6 +119,7 @@ ubuntu_16.04_amd64_deb:
- cd ..
- echo "" | sudo -S DEBIAN_FRONTEND=noninteractive dpkg --force-all -i pd-l2ork*.deb > /dev/null
- cd scripts
- valgrind pd-l2ork -noprefs -nostdpath -nogui -nrt -noaudio -send "init dollarzero \$0" regression_tests.pd
- valgrind pd-l2ork -noprefs -nostdpath -nogui -nrt -noaudio external-tests.pd
artifacts:
name: "$CI_RUNNER_DESCRIPTION"
......@@ -131,12 +137,14 @@ osx_amd64_dmg:
- cd l2ork_addons
- ./tar_em_up.sh -X
- cd ../scripts
- ../packages/darwin_app/build/*.app/Contents/Resources/app.nw/bin/pd-l2ork -noprefs -nostdpath -nogui -noaudio -send "init dollarzero \$0" regression_tests.pd
- ../packages/darwin_app/build/*.app/Contents/Resources/app.nw/bin/pd-l2ork -noprefs -nostdpath -nogui -noaudio external-tests.pd
artifacts:
name: "$CI_RUNNER_DESCRIPTION"
expire_in: 1 day
paths:
- Pd*.dmg
osx_10.8_amd64_dmg:
tags:
- osx
......@@ -147,6 +155,7 @@ osx_10.8_amd64_dmg:
- cd l2ork_addons
- osx_version=10.8 ./tar_em_up.sh -X
- cd ../scripts
- ../packages/darwin_app/build/*.app/Contents/Resources/app.nw/bin/pd-l2ork -noprefs -nostdpath -nogui -noaudio -send "init dollarzero \$0" regression_tests.pd
- ../packages/darwin_app/build/*.app/Contents/Resources/app.nw/bin/pd-l2ork -noprefs -nostdpath -nogui -noaudio external-tests.pd
artifacts:
name: "$CI_RUNNER_DESCRIPTION"
......@@ -163,6 +172,7 @@ windows_i386_installer:
- cd l2ork_addons
- ./tar_em_up.sh -Z
- cd ../scripts
- ../packages/win32_inno/build/bin/pd.exe -noprefs -nostdpath -nogui -noaudio -send "init dollarzero \$0" regression_tests.pd
- ../packages/win32_inno/build/bin/pd.exe -noprefs -nostdpath -nogui -noaudio external-tests.pd
artifacts:
name: "$CI_RUNNER_DESCRIPTION"
......@@ -180,6 +190,7 @@ debian_stable_arm_deb:
- cd l2ork_addons
- ./tar_em_up.sh -r
- cd ..
- valgrind pd-l2ork -noprefs -nostdpath -nogui -nrt -noaudio -send "init dollarzero \$0" scripts/regression_tests.pd
- valgrind pd-l2ork -noprefs -nostdpath -nogui -nrt -noaudio scripts/external-tests.pd
artifacts:
name: "$CI_RUNNER_DESCRIPTION"
......
#N canvas 219 60 984 603 10;
#X msg 272 255 10000;
#X obj 272 277 until;
#X obj 207 206 t b b b;
#X obj 211 256 realtime;
#X obj 211 329 +;
#X obj 211 384 /;
#X obj 211 299 t a b;
#X obj 238 330 f;
#X obj 238 352 + 1;
#X obj 211 351 t a;
#X obj 173 384 t a;
#X obj 196 464 f;
#X obj 68 464 f;
#X obj 180 176 t b b;
#X obj 68 382 t b b;
#X floatatom 68 486 5 0 0 0 worst - -, f 5;
#X floatatom 196 486 5 0 0 0 average - -, f 5;
#X obj 125 384 moses;
#X obj 243 26 tgl 15 0 empty empty empty 17 7 0 10 -262144 -1 -1 0
1;
#X obj 243 46 metro 150;
#X obj 272 299 list append one two three;
#X text 312 92 nothing;
#X text 632 92 \$0;
#X text 605 138 set/unsetcurrent: 13/36;
#X text 285 138 set/unsetcurrent:11.5/32;
#X text 792 92 list one two three four \$0 \$1;
#X text 765 138 set/unsetcurrent: 17/37;
#X text 765 118 nothing: 14.5/35.5;
#X text 605 118 nothing: 8/17;
#X text 285 118 nothing: 5/12;
#X text 472 92 42;
#X text 445 118 nothing: 7/17;
#X text 445 138 set/unsetcurrent:14/35;
#X text 285 158 cached glist:4.5/11;
#X text 605 158 cached glist: 8.7/19;
#X text 445 158 cached glist:5.6/15;
#X text 765 158 cached glist: 13/32;
#X text 285 178 semicolon fix: 4/12.5;
#X text 445 178 semicolon fix: 6/17;
#X text 605 178 cached glist: 6.5/19;
#X msg 272 321 list one two three four \$0 \$1;
#X text 765 178 cached glist: 13/35;
#X connect 0 0 1 0;
#X connect 1 0 20 0;
#X connect 2 0 3 1;
#X connect 2 1 0 0;
#X connect 2 2 3 0;
#X connect 3 0 6 0;
#X connect 4 0 9 0;
#X connect 5 0 11 1;
#X connect 6 0 4 0;
#X connect 6 0 17 0;
#X connect 6 1 7 0;
#X connect 7 0 8 0;
#X connect 8 0 7 1;
#X connect 8 0 5 1;
#X connect 9 0 5 0;
#X connect 9 0 4 1;
#X connect 10 0 12 1;
#X connect 10 0 17 1;
#X connect 11 0 16 0;
#X connect 12 0 15 0;
#X connect 13 0 14 0;
#X connect 13 1 2 0;
#X connect 14 0 12 0;
#X connect 14 1 11 0;
#X connect 17 1 10 0;
#X connect 18 0 19 0;
#X connect 19 0 13 0;
#X connect 20 0 40 0;
#N canvas 3 60 508 425 10;
#X msg 21 41 \; \$1;
#X msg 21 131 \; \$0;
#X msg 21 220 42;
#X msg 21 251 \; \$1;
#X text 21 11 1) dollar target is out of range;
#X text 21 101 2) dollar target is dollar zero;
#X text 21 191 3) dollar target isn't a symbol;
#X text 251 11 4) not enough args to dollsym;
#X msg 251 67 \; \$1-foo;
#X msg 251 36 bang;
#X text 299 59 ERROR: double doll sign;
#X msg 251 130 bang;
#X text 251 101 5) no such object;
#X msg 251 161 \; 42;
#X msg 251 230 bang;
#X msg 251 261 \; foo \$1;
#X obj 329 228 r foo;
#X obj 329 250 == 0;
#X floatatom 329 272 5 0 0 0 - - -, f 5;
#X msg 251 340 bang;
#X text 251 201 6) dollar argument number out of range;
#X text 251 311 7) dollsym argument number out of range;
#X obj 379 360 sel 0-foo;
#X obj 379 338 r seven;
#X msg 251 371 \; seven symbol \$1-foo;
#X obj 430 382 bng 15 250 50 0 empty empty empty 17 7 0 10 -262144
-1 -1;
#X connect 2 0 3 0;
#X connect 9 0 8 0;
#X connect 11 0 13 0;
#X connect 14 0 15 0;
#X connect 16 0 17 0;
#X connect 17 0 18 0;
#X connect 19 0 24 0;
#X connect 22 1 25 0;
#X connect 23 0 22 0;
#N canvas 311 60 867 603 10;
#X msg 712 255 10000;
#X obj 712 277 until;
#X obj 712 299 f;
#X obj 712 380 f;
#X obj 742 380 f;
#X msg 620 154 bang;
#X obj 647 206 t b b b;
#X obj 651 256 realtime;
#X obj 651 329 +;
#X obj 651 384 /;
#X obj 651 299 t a b;
#X obj 678 330 f;
#X obj 678 352 + 1;
#X obj 651 351 t a;
#X obj 613 384 t a;
#X obj 636 464 f;
#X obj 508 464 f;
#X obj 620 176 t b b;
#X obj 508 382 t b b;
#X floatatom 508 486 5 0 0 0 worst - -, f 5;
#X floatatom 636 486 5 0 0 0 average - -, f 5;
#X obj 565 384 moses;
#X msg 272 255 10000;
#X obj 272 277 until;
#X obj 272 299 f;
#X obj 272 380 f;
#X obj 302 380 f;
#X msg 180 154 bang;
#X obj 207 206 t b b b;
#X obj 211 256 realtime;
#X obj 211 329 +;
#X obj 211 384 /;
#X obj 211 299 t a b;
#X obj 238 330 f;
#X obj 238 352 + 1;
#X obj 211 351 t a;
#X obj 173 384 t a;
#X obj 196 464 f;
#X obj 68 464 f;
#X obj 180 176 t b b;
#X obj 68 382 t b b;
#X floatatom 68 486 5 0 0 0 worst - -, f 5;
#X floatatom 196 486 5 0 0 0 average - -, f 5;
#X obj 125 384 moses;
#X obj 243 26 tgl 15 0 empty empty empty 17 7 0 10 -262144 -1 -1 0
1;
#X obj 243 46 metro 150;
#X obj 293 535 /;
#X floatatom 293 557 5 0 0 0 ratio - -, f 5;
#X obj 243 68 t b b;
#X obj 272 321 t f f;
#X connect 0 0 1 0;
#X connect 1 0 2 0;
#X connect 2 0 3 0;
#X connect 2 0 4 0;
#X connect 5 0 17 0;
#X connect 6 0 7 1;
#X connect 6 1 0 0;
#X connect 6 2 7 0;
#X connect 7 0 10 0;
#X connect 8 0 13 0;
#X connect 9 0 15 1;
#X connect 10 0 8 0;
#X connect 10 0 21 0;
#X connect 10 1 11 0;
#X connect 11 0 12 0;
#X connect 12 0 11 1;
#X connect 12 0 9 1;
#X connect 13 0 9 0;
#X connect 13 0 8 1;
#X connect 14 0 16 1;
#X connect 14 0 21 1;
#X connect 15 0 20 0;
#X connect 15 0 46 1;
#X connect 16 0 19 0;
#X connect 17 0 18 0;
#X connect 17 1 6 0;
#X connect 18 0 16 0;
#X connect 18 1 15 0;
#X connect 21 1 14 0;
#X connect 22 0 23 0;
#X connect 23 0 24 0;
#X connect 24 0 49 0;
#X connect 27 0 39 0;
#X connect 28 0 29 1;
#X connect 28 1 22 0;
#X connect 28 2 29 0;
#X connect 29 0 32 0;
#X connect 30 0 35 0;
#X connect 31 0 37 1;
#X connect 32 0 30 0;
#X connect 32 0 43 0;
#X connect 32 1 33 0;
#X connect 33 0 34 0;
#X connect 34 0 33 1;
#X connect 34 0 31 1;
#X connect 35 0 31 0;
#X connect 35 0 30 1;
#X connect 36 0 38 1;
#X connect 36 0 43 1;
#X connect 37 0 42 0;
#X connect 37 0 46 0;
#X connect 38 0 41 0;
#X connect 39 0 40 0;
#X connect 39 1 28 0;
#X connect 40 0 38 0;
#X connect 40 1 37 0;
#X connect 43 1 36 0;
#X connect 44 0 45 0;
#X connect 45 0 48 0;
#X connect 46 0 47 0;
#X connect 48 0 27 0;
#X connect 48 1 5 0;
#X connect 49 0 25 0;
#X connect 49 1 26 0;
#N canvas 3 60 749 617 12;
#X obj 345 301 r \$0-result;
#X obj 345 326 route 0;
#X obj 453 470 print failure;
#X obj 430 336 tgl 28 0 empty empty Print_All_Results 31 11 0 12 -262144
-1 -1 1 1;
#X obj 159 149 bng 31 250 50 0 empty empty Run_all 39 13 0 12 -262144
-1 -1;
#X obj 56 25 r init;
#X obj 345 191 route dollarzero;
#X obj 345 411 t b a;
#X obj 345 541 s pd;
#X obj 56 120 trigger bang bang anything;
#X msg 56 145 gui;
#X obj 56 170 pdinfo;
#X obj 56 195 sel 0;
#X obj 56 245 s pd;
#X msg 345 516 quit 1;
#X msg 56 220 quit;
#X obj 145 191 rtest msg_dollarzero;
#X obj 145 246 rtest msg_dollarzero_semi;
#X obj 145 302 rtest msg_click;
#X obj 345 216 rtest binbuf_dollarzero;
#X msg 345 440 gui;
#X obj 345 465 pdinfo;
#X obj 345 490 sel 0;
#X text 117 25 <- we start Pd with the -send "init etc." flag. This
will automatically start the tests and allow us to send a comma-separated
list of messages which will be evaluated by Pd without a target. This
is handy for some binbuf tests.;
#X text 345 120 To run this test we use the following startup-flag:
;
#X text 345 150 -send "init dollarzero \$0";
#X text 536 150 <- we have to escape the arg;
#X text 556 190 escape it in a comment.;
#X text 556 170 in bash but we can't;
#X obj 391 374 spigot;
#X obj 407 440 route 1;
#X obj 407 495 print success;
#X connect 0 0 1 0;
#X connect 1 0 7 0;
#X connect 1 1 29 0;
#X connect 3 0 29 1;
#X connect 4 0 16 0;
#X connect 5 0 9 0;
#X connect 6 0 19 0;
#X connect 7 0 20 0;
#X connect 7 1 30 0;
#X connect 9 0 10 0;
#X connect 9 1 16 0;
#X connect 9 2 6 0;
#X connect 10 0 11 0;
#X connect 11 0 12 0;
#X connect 12 0 15 0;
#X connect 14 0 8 0;
#X connect 15 0 13 0;
#X connect 16 0 17 0;
#X connect 17 0 18 0;
#X connect 20 0 21 0;
#X connect 21 0 22 0;
#X connect 22 0 14 0;
#X connect 29 0 7 0;
#X connect 30 0 31 0;
#X connect 30 1 2 0;
#N canvas 144 129 582 396 12;
#X obj 36 25 inlet;
#X obj 36 225 outlet;
#X obj 36 50 route float;
#X obj 36 116 f 1;
#X obj 36 75 select 0;
#X obj 89 115 t b a;
#X obj 121 152 list prepend dollarzero in a binbuf with no target should
expand to "0". In this test it expanded to;
#X obj 89 152 f 0;
#X obj 36 195 list append;
#X connect 0 0 2 0;
#X connect 2 0 4 0;
#X connect 2 1 5 0;
#X connect 3 0 8 0;
#X connect 4 0 3 0;
#X connect 4 1 5 0;
#X connect 5 0 7 0;
#X connect 5 1 6 0;
#X connect 6 0 8 1;
#X connect 7 0 8 0;
#X connect 8 0 1 0;
#N canvas 34 75 582 396 12;
#X obj 36 25 inlet;
#X msg 152 97 click 0 0 0 0 0;
#X msg 152 122 \$1;
#X obj 36 50 trigger bang bang;
#X obj 36 97 f 0;
#X msg 36 122 \$1;
#X obj 36 154 ==;
#X obj 36 229 outlet;
#X obj 36 188 list append clicking a message box should send a single
"0" as an argument;
#X connect 0 0 3 0;
#X connect 1 0 2 0;
#X connect 2 0 6 1;
#X connect 3 0 4 0;
#X connect 3 1 1 0;
#X connect 4 0 5 0;
#X connect 5 0 6 0;
#X connect 6 0 8 0;
#X connect 8 0 7 0;
#N canvas 12 147 582 396 12;
#X obj 36 25 inlet;
#X obj 36 54 t b b;
#X obj 36 90 f \$0;
#X msg 93 90 \$0;
#X obj 36 324 outlet;
#X obj 36 197 ==;
#X obj 36 292 list append;
#X obj 68 197 list;
#X obj 36 115 t a a;
#X obj 93 115 t a a;
#X msg 68 224 dollarzero should have the same value in a message box
as it does in an object box. In this test object box dollarzero was
\$1 and msg box dollarzero was \$2;
#X connect 0 0 1 0;
#X connect 1 0 2 0;
#X connect 1 1 3 0;
#X connect 2 0 8 0;
#X connect 3 0 9 0;
#X connect 5 0 6 0;
#X connect 6 0 4 0;
#X connect 7 0 10 0;
#X connect 8 0 5 0;
#X connect 8 1 7 0;
#X connect 9 0 5 1;
#X connect 9 1 7 1;
#X connect 10 0 6 1;
#N canvas 57 140 582 396 12;
#X obj 36 25 inlet;
#X obj 36 54 t b b;
#X obj 36 264 outlet;
#X obj 36 195 f 0;
#X obj 54 135 r \$0-dollsym;
#X msg 68 90 \; \$0-dollsym bang;
#X obj 54 161 f 1;
#X obj 36 223 list append dollarzero should expand properly in a symbol
that follows a semicolon;
#X connect 0 0 1 0;
#X connect 1 0 3 0;
#X connect 1 1 5 0;
#X connect 3 0 7 0;
#X connect 4 0 6 0;
#X connect 6 0 3 1;
#X connect 7 0 2 0;
#N canvas 212 151 506 469 12;
#X obj 36 60 inlet;
#X obj 36 416 outlet;
#X obj 68 387 send;
#X obj 93 282 loadbang;
#X msg 93 308 dollarzero;
#X obj 93 334 canvasinfo 1;
#X msg 93 359 symbol \$1-result;
#X obj 106 63 bng 20 250 50 0 empty empty Run 21 8 0 12 -262144 -1
-1;
#X obj 68 124 regression_tests/\$1;
#X obj 36 85 t b a;
#X obj 68 149 list split 1;
#X obj 68 244 list;
#X text 260 201 <- insert "test_name:" here;
#X obj 219 124 loadbang;
#X obj 219 149 list append \$1;
#X msg 219 174 \$1:;
#X obj 108 211 list prepend;
#X obj 156 63 bng 20 250 50 0 empty empty Show_Test 21 8 0 12 -262144
-1 -1;
#N canvas 378 161 450 369 vis 0;
#X obj 118 84 list append \$1;
#X obj 86 34 inlet;
#X obj 93 146 send;
#X msg 58 102 vis 1;
#X obj 86 59 t b b;
#X msg 118 109 symbol pd-\$1.pd;
#X connect 0 0 5 0;
#X connect 1 0 4 0;
#X connect 3 0 2 0;
#X connect 4 0 3 0;
#X connect 4 1 0 0;
#X connect 5 0 2 1;
#X restore 156 94 pd vis;
#X connect 0 0 9 0;
#X connect 3 0 4 0;
#X connect 4 0 5 0;
#X connect 5 0 6 0;
#X connect 6 0 2 1;
#X connect 7 0 8 0;
#X connect 8 0 10 0;
#X connect 9 0 1 0;
#X connect 9 1 8 0;
#X connect 10 0 11 0;
#X connect 10 1 16 0;
#X connect 11 0 2 0;
#X connect 13 0 14 0;
#X connect 14 0 15 0;
#X connect 15 0 16 1;
#X connect 16 0 11 1;
#X connect 17 0 18 0;
#X coords 0 -1 1 1 165 46 1 99 42;
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment