Book a Demo

Author Topic: Automate add-in testing  (Read 3416 times)

Risco

  • EA Novice
  • *
  • Posts: 2
  • Karma: +0/-0
    • View Profile
Automate add-in testing
« on: March 10, 2020, 08:17:17 pm »
Hello,

We are developing an Add-in and we would like to start automating the testing of the Add-in. It is being developed in C# using the framework .NET

I saw this post from 13 years ago: https://sparxsystems.com/forums/smf/index.php?topic=1351.0 , they talk about RhinoMocks and some other solutions.

I was wondering if nowadays there are out there any other solutions? What are you guys using for testing your add-in?

Geert Bellekens

  • EA Guru
  • *****
  • Posts: 13523
  • Karma: +574/-33
  • Make EA work for YOU!
    • View Profile
    • Enterprise Architect Consultant and Value Added Reseller
Re: Automate add-in testing
« Reply #1 on: March 10, 2020, 08:27:22 pm »
I'm using manual testing only  :-\

Geert

timoc

  • EA User
  • **
  • Posts: 201
  • Karma: +14/-0
    • View Profile
Re: Automate add-in testing
« Reply #2 on: March 11, 2020, 12:57:00 am »
I played around with externally testing Javascript with mock's etc. Someone (in the forums?) ported the jsunity javascript unit test framework (below), but i have not used it yet.

If you have any success with it, or any unit test framework - please post it to the forum!

I for one am very interested in some reasonably standard way to test an addin before it has the opportunity to mess with anything valuable. Especially true for fluctuations in EA Javscript and EA object model runtime environments.

Code: [Select]
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/**
 * jsUnity Universal JavaScript Testing Framework v0.6
 * http://jsunity.com/
 *
 * Copyright (c) 2009 Ates Goral
 * Licensed under the MIT license.
 * http://www.opensource.org/licenses/mit-license.php
 *
 * Modified by Steve Savage to work under Sparx EA as a JSCript script
 */
 
if(typeof(jsUnity) == "undefined")
{
jsUnity = (function () {

function fmt(str) {
var a = Array.prototype.slice.call(arguments, 1);
return str.replace(/\?/g, function () { return a.shift(); });
}

function hash(v) {
if (v instanceof Object) {
var arr = [];

for (var p in v) {
arr.push(p);
arr.push(hash(v[p]));   
}

return arr.join("#");
} else {
return String(v);
}
}

var defaultAssertions = {
assertException: function (fn, message) {
try {
fn instanceof Function && fn();
} catch (e) {
return;
}

throw fmt("?: (?) does not raise an exception or not a function",
message || "assertException", fn);
},

assertTrue: function (actual, message) {
if (!actual) {
throw fmt("?: (?) does not evaluate to true",
message || "assertTrue", actual);
}
},

assertFalse: function (actual, message) {
if (actual) {
throw fmt("?: (?) does not evaluate to false",
message || "assertFalse", actual);
}
},

assertIdentical: function (expected, actual, message) {
if (expected !== actual) {
throw fmt("?: (?) is not identical to (?)",
message || "assertIdentical", actual, expected);
}
},

assertNotIdentical: function (expected, actual, message) {
if (expected === actual) {
throw fmt("?: (?) is identical to (?)",
message || "assertNotIdentical", actual, expected);
}
},

assertEqual: function (expected, actual, message) {
if (hash(expected) != hash(actual)) {
throw fmt("?: (?) is not equal to (?)",
message || "assertEqual", actual, expected);
}
},

assertNotEqual: function (expected, actual, message) {
if (hash(expected) == hash(actual)) {
throw fmt("?: (?) is equal to (?)",
message || "assertNotEqual", actual, expected);
}
},

assertMatch: function (re, actual, message) {
if (!re.test(actual)) {
throw fmt("?: (?) does not match (?)",
message || "assertMatch", actual, re);
}
},

assertNotMatch: function (re, actual, message) {
if (re.test(actual)) {
throw fmt("?: (?) matches (?)",
message || "assertNotMatch", actual, re);
}
},

assertTypeOf: function (typ, actual, message) {
if (typeof actual !== typ) {
throw fmt("?: (?) is not of type (?)",
message || "assertTypeOf", actual, typ);
}
},

assertNotTypeOf: function (typ, actual, message) {
if (typeof actual === typ) {
throw fmt("?: (?) is of type (?)",
message || "assertNotTypeOf", actual, typ);
}
},

assertInstanceOf: function (cls, actual, message) {
if (!(actual instanceof cls)) {
throw fmt("?: (?) is not an instance of (?)",
message || "assertInstanceOf", actual, cls);
}
},

assertNotInstanceOf: function (cls, actual, message) {
if (actual instanceof cls) {
throw fmt("?: (?) is an instance of (?)",
message || "assertNotInstanceOf", actual, cls);
}
},

assertNull: function (actual, message) {
if (actual !== null) {
throw fmt("?: (?) is not null",
message || "assertNull", actual);
}
},

assertNotNull: function (actual, message) {
if (actual === null) {
throw fmt("?: (?) is null",
message || "assertNotNull", actual);
}
},

assertUndefined: function (actual, message) {
if (actual !== undefined) {
throw fmt("?: (?) is not undefined",
message || "assertUndefined", actual);
}
},

assertNotUndefined: function (actual, message) {
if (actual === undefined) {
throw fmt("?: (?) is undefined",
message || "assertNotUndefined", actual);
}
},

assertNaN: function (actual, message) {
if (!isNaN(actual)) {
throw fmt("?: (?) is not NaN",
message || "assertNaN", actual);
}
},

assertNotNaN: function (actual, message) {
if (isNaN(actual)) {
throw fmt("?: (?) is NaN",
message || "assertNotNaN", actual);
}
},

fail: function (message) {
throw message || "fail";
}
};

function plural(cnt, unit) {
return cnt + " " + unit + (cnt == 1 ? "" : "s");
}

function splitFunction(fn) {
var tokens =
/^[\s\r\n]*function[\s\r\n]*([^\(\s\r\n]*?)[\s\r\n]*\([^\)\s\r\n]*\)[\s\r\n]*\{((?:[^}]*\}?)+)\}[\s\r\n]*$/
.exec(fn);

if (!tokens) {
throw "Invalid function.";
}

return {
name: tokens[1].length ? tokens[1] : undefined,
body: tokens[2]
};
}

var probeOutside = function () {
try {
return eval(
[ "typeof ", " === \"function\" && ", "" ].join(arguments[0]));
} catch (e) {
return false;
}
};

function parseSuiteString(str) {
var obj = {};

var probeInside = new Function(
splitFunction(probeOutside).body + str);

var tokenRe = /(\w+)/g; // todo: wiser regex
var tokens;

while ((tokens = tokenRe.exec(str))) {
var token = tokens[1];
var fn;

if (!obj[token]
&& (fn = probeInside(token))
&& fn != probeOutside(token)) {

obj[token] = fn;
}
}

return parseSuiteObject(obj);
}

function parseSuiteFunction(fn) {
var fnParts = splitFunction(fn);
var suite = parseSuiteString(fnParts.body);

suite.suiteName = fnParts.name;

return suite;
}

function parseSuiteArray(tests) {
var obj = {};

for (var i = 0; i < tests.length; i++) {
var item = tests[i];

if (!obj[item]) {
switch (typeof item) {
case "function":
var fnParts = splitFunction(item);
obj[fnParts.name] = item;
break;
case "string":
var fn;

if (fn = probeOutside(item)) {
obj[item] = fn;
}
}
}
}

return parseSuiteObject(obj);
}

function parseSuiteObject(obj) {
var suite = new jsUnity.TestSuite(obj.suiteName, obj);

for (var name in obj) {
if (obj.hasOwnProperty(name)) {
var fn = obj[name];

if (typeof fn === "function") {
if (/^test/.test(name)) {
suite.tests.push({ name: name, fn: fn });
} else if (/^setUp|tearDown$/.test(name)) {
suite[name] = fn;
}
}
}
}

return suite;
}

return {
TestSuite: function (suiteName, scope) {
this.suiteName = suiteName;
this.scope = scope;
this.tests = [];
this.setUp = undefined;
this.tearDown = undefined;
},

TestResults: function () {
this.suiteName = undefined;
this.total = 0;
this.passed = 0;
this.failed = 0;
this.duration = 0;
},

assertions: defaultAssertions,

env: {
defaultScope: this,

getDate: function () {
return new Date();
}
},

attachAssertions: function (scope) {
scope = scope || this.env.defaultScope;

for (var fn in jsUnity.assertions) {
scope[fn] = jsUnity.assertions[fn];
}
},

log: function (s) {
Session.Output(s);
},

error: function (s) { this.log("[ERROR] " + s); },

compile: function (v) {
if (v instanceof jsUnity.TestSuite) {
return v;
} else if (v instanceof Function) {
return parseSuiteFunction(v);
} else if (v instanceof Array) {
return parseSuiteArray(v);
} else if (v instanceof Object) {
return parseSuiteObject(v);
} else if (typeof v === "string") {
return parseSuiteString(v);
} else {
throw "Argument must be a function, array, object, string or "
+ "TestSuite instance.";
}
},

run: function () {
var results = new jsUnity.TestResults();

var suiteNames = [];
var start = jsUnity.env.getDate();

for (var i = 0; i < arguments.length; i++) {
try {
var suite = jsUnity.compile(arguments[i]);
} catch (e) {
this.error("Invalid test suite: " + e);
return false;
}

var cnt = suite.tests.length;

this.log("*******************************************");
this.log("Running : "
+ (suite.suiteName || "unnamed test suite"));
this.log(plural(cnt, "test") + " found");
this.log("-------------------------------------------");

suiteNames.push(suite.suiteName);
results.total += cnt;

for (var j = 0; j < cnt; j++) {
var test = suite.tests[j];

try {
suite.setUp && suite.setUp();
test.fn.call(suite.scope);
suite.tearDown && suite.tearDown();

results.passed++;

this.log("UnitTest: [PASSED] " + test.name);
} catch (e) {
suite.tearDown && suite.tearDown();

this.log("UnitTest: [FAILED] " + test.name + ": " + e);
}
}
}

results.suiteName = suiteNames.join(",");
results.failed = results.total - results.passed;
results.duration = jsUnity.env.getDate() - start;

this.log(plural(results.passed, "test") + " passed");
this.log(plural(results.failed, "test") + " failed");
this.log(plural(results.duration, "millisecond") + " elapsed");
this.log("*******************************************");

return results;
}
};
})();
}

PeteC

  • EA User
  • **
  • Posts: 91
  • Karma: +1/-0
    • View Profile
Re: Automate add-in testing
« Reply #3 on: March 11, 2020, 07:46:51 pm »
There has been a similar post on here suggesting AutoIt.

I tried it for testing my MDG, but have gone back to manual testing. There is nothing wrong with AutoIt and in some scenarios it would be fine. Issues are the time to develop the test (vs manual testing) and the maintainability - as soon as the next version of EA moves something around then recorded mouse clicks are in the wrong place, menus change (so even trying to automate through keyboard selection of menus changes).