Rick

Rick
Rick

Saturday, April 30, 2016

Creating your own custom DSL for config files with Konf (example looking up Mesos Ports or Docker Ports)

Konf is a Java configuration system. You can use it to easily create your own config DSLs.
At times it is helpful to add some configuration logic to a config file. As long as you keep the logic to config small and only put in config logic. The config logic should not be complicated.
The config logic forms a DSL for your configuration. If you do not need this then use JSON or YAML for your config. We find it very useful for working in EC2, Jenkins, local dev boxes, Nomad, Heroku and Mesosphere.
Here is an example that figures out port numbers running in Mesosphere.

mycompany-config-utils.js example config javascript

var createLogger = Java.type("org.slf4j.LoggerFactory").getLogger;

var log = createLogger("config.log");

function mesosPortAt(index, defaultPort) {
  var fromMesos = env("PORT" + index);
  var portReturned = fromMesos ? parseInt(fromMesos) : defaultPort;
  log.info("Mesos Port At " + index + " was " + portReturned +
            " default was {}" + defaultPort);
  return portReturned;
}
Then you can use this config "DSL" from your config files. The example todo-service-development.js uses the config logic functions from the previous example, namely,mesosPortAt and createLogger (development deploy).

todo-service-development.js

var config = {

  platform: {

    statsd: "udp://" + getDockerHost() + ":8125",

    servicePort: mesosPortAt(0, 8080),
    adminPort: mesosPortAt(1, 9090),

    vertxOptions: {
      clustered: false
    },

    discovery: {
      providers: ["docker:http://" + getDockerHost() + ":2375"]
    }
  },

  todoService: {

    recordTodoTimeout: "30 seconds",
    recordTodoListTimeout: "10 seconds",
    circuitBreakerCheckInterval: '10s',
    maxErrorsPerCircuitBreakerCheck: 10,
    checkCassandraConnectionInterval: "10s",

    cassandra: {
      uri: "discovery:docker:///cassandra?containerPort=9042",
      replicationFactor: 1
    }

  }

};

function getDockerHost() {
  return isMacOS() && !env("DOCKER_HOST")
    ? "192.168.99.100"
    : dockerHostOrDefault("localhost");
}

log.info("STAGING DEVELOPMENT");
log.info("impressions DEPLOYMENT_ENVIRONMENT {} ", env("DEPLOYMENT_ENVIRONMENT"));
log.info("impressions DOCKER_HOST {} ", env("DOCKER_HOST"));
log.info("impressions PORT_0 {} ", env("PORT0"));
log.info("impressions PORT_1 {} ", env("PORT1"));
Notice we added getDockerHost to our dev config file. These config files would exist in your deployment jar files.
Your production version can use just your "standard" config functions as follows.

todo-service-production.js

var config = {

  platform: {

    statsd: "udp://statsd1.ops.prod.somecompany.net:8086",

    servicePort: mesosPortAt(0, 8080),
    adminPort: mesosPortAt(1, 9090),


    vertxOptions: {
      clustered: false
    },

    discovery: {
      providers: ["dns://ns-660.aawsdns-13.net:53", 
                  "dns://ns-1399.aawsdns-36.org:53", 
                  "dns://ns-511.aawsdns-63.com:53"]
    }
  },

  todoService: {

    recordTodoTimeout: "30 seconds",
    recordTodoListTimeout: "10 seconds",
    circuitBreakerCheckInterval: '10s',
    maxErrorsPerCircuitBreakerCheck: 10,
    checkCassandraConnectionInterval: "10s",

    cassandra: {
      uri: "discovery:dns:A:///cassandra1.ds.prod.rbmhops.net?port=9042",
      replicationFactor: 1
    }

  }

};


log.info("STAGING PRODUCTION");
log.info("impressions DEPLOYMENT_ENVIRONMENT {} ", env("DEPLOYMENT_ENVIRONMENT"));
log.info("impressions DOCKER_HOST {} ", env("DOCKER_HOST"));
log.info("impressions PORT_0 {} ", env("PORT0"));
log.info("impressions PORT_1 {} ", env("PORT1"));
This was your production and QA config are locked down, but your dev config is more flexible to accommodate different dev environments (Linux, MacOSX, and Windows).
To load your base config utils, create a Java utility jar that your apps depend on as follows.
package io.advantageous.platform.config;


import io.advantageous.config.Config;
import io.advantageous.config.ConfigLoader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.concurrent.atomic.AtomicReference;

/**
 * Convenient utility methods for handling env and config.
 *
 * @author Geoff Chandler
 * @author Rick Hightower
 */
public final class ConfigUtils {

    private static final String DEPLOYMENT_ENVIRONMENT;
    private static final Logger logger = LoggerFactory.getLogger(ConfigUtils.class);
    private static AtomicReference<Config> rootConfig = new AtomicReference<>();

    static {
        final String env = System.getenv("DEPLOYMENT_ENVIRONMENT");
        DEPLOYMENT_ENVIRONMENT = env == null ? "development" : env.toLowerCase();
    }

    private ConfigUtils() {
        throw new IllegalStateException("config utils is not to be instantiated.");
    }

    private static Config loadRootConfig() {
        // Load the right config for the right environment.
        final String resourceName = String.format("config-%s.js", DEPLOYMENT_ENVIRONMENT);
        //Load your config utils
        final Config load = ConfigLoader.load("mycompany-config-utils.js", resourceName);

        if (!rootConfig.compareAndSet(null, load)) {
            logger.warn("Config was set, and and you can't overwrite it. {}", resourceName);
        }
        return load;
    }

    public static Config getConfig(final String basePath) {
        if (rootConfig.get() == null) {
            loadRootConfig();
        }
        return rootConfig.get().getConfig(basePath);
    }

}
That is it. Now you can write your own config DSLs. Happy coding!
Konf has the following built-in config functions.
var env = Java.type("java.lang.System").getenv;
var uri = Java.type("java.net.URI").create;
var system = Java.type("java.lang.System");
var duration = Java.type("java.time.Duration");

/** To store private vars. */
var konf = {
  osNameInternal : system.getProperty("os.name").toLowerCase()
};

function seconds(unit) {
  return duration.ofSeconds(unit);
}

function minutes(unit) {
  return duration.ofMinutes(unit);
}

function hours(unit) {
  return duration.ofHours(unit);
}

function days(unit) {
  return duration.ofDays(unit);
}


function millis(unit) {
  return duration.ofMillis(unit);
}


function milliseconds(unit) {
  return duration.ofMillis(unit);
}

function sysProp(prop) {
  return system.getProperty(prop);
}

function sysPropOrDefault(prop, defaultValue) {
  return system.getProperty(prop, defaultValue.toString());
}

function isWindowsOS() {
  return (konf.osNameInternal.indexOf("win") >= 0);
}

function isMacOS() {
  return (konf.osNameInternal.indexOf("mac") >= 0);
}

function isUnix() {
  var OS = konf.osNameInternal;
  return (OS.indexOf("nix") >= 0 || OS.indexOf("nux") >= 0 || OS.indexOf("aix") > 0 );
}


function isLinux() {
  return konf.osNameInternal.indexOf("linux") >= 0;
}


function  isSolaris() {
  return (konf.osNameInternal.indexOf("sunos") >= 0);
}

Friday, April 29, 2016

QBit Resourceful RESTful Microservices

You can build resourceful REST URIs using @RequestMapping from QBit microservices lib.
Typically you use HTTP GET to get a list of something and if you are returning more than one then the URI path should end with / as follows:

@RequestMapping("/department/")

    @RequestMapping("/department/")
    public List<Department> getDepartments() {
To add to a list, you would use a PUT or a POST. PUT is generally used for updates, and POST is used to create. If you were editing an object at a give ID you would use a PUT, but if you were adding a new item to a list, you would use a PUT to update a list or a POST to create an item. Dealers choice.

@RequestMapping(value = "/department/{departmentId}/", method = RequestMethod.POST)

@RequestMapping(value = "/department/{departmentId}/", method = RequestMethod.POST)
public boolean addDepartment(@PathVariable("departmentId") Integer departmentId,
                                  final Department department) {
You could easily get into a long drawn out argument about which to use PUT or POST in the above scenario, because you are updating the list (adding an item to it), but you are creating a department. Just remember POST for create, and PUT for update.
To get a single employee, you could use a path param. Some say that path params are nice for people and search engines, I say ok dude.

Using a path param @RequestMapping(value = "/department/{departmentId}/employee/{employeeId}", method = RequestMethod.GET)

    @RequestMapping(value = "/department/{departmentId}/employee/{employeeId}", method = RequestMethod.GET)
    public Employee getEmployee(@PathVariable("departmentId") Integer departmentId,
                                @PathVariable("employeeId") Long employeeId) {
HTTP GET is the default, but you can specify it as we did above.
There are other annotations that work the same way except the HTTP method is in the name of the param.
  • @POST
  • @PUT
  • @GET
You could rewrite the last examples like this.

Using the @POST, @PUT, @GET

    @GET("/department/")
    public List<Department> getDepartments() {

...
    @POST(value = "/department/{departmentId}/")
    public boolean addDepartment(@PathVariable("departmentId") Integer departmentId,
                                  final Department department) {


...


    @GET(value = "/department/{departmentId}/employee/{employeeId}")
    public Employee getEmployee(@PathVariable("departmentId") Integer departmentId,
                                @PathVariable("employeeId") Long employeeId) {
Making the method name also be the annotation name makes things a bit more compact, and a bit easier to read.
Here are some more Resourceful REST examples in QBit to ponder.

Resource based RESTful API for Microservices

package com.mammatustech.hr;



import io.advantageous.qbit.annotation.*;

import java.util.*;

@RequestMapping("/hr")
public class HRService {

    final Map<Integer, Department> departmentMap = new HashMap<>();


    @RequestMapping("/department/")
    public List<Department> getDepartments() {
        return new ArrayList<>(departmentMap.values());
    }

    @RequestMapping(value = "/department/{departmentId}/", method = RequestMethod.POST)
    public boolean addDepartment(@PathVariable("departmentId") Integer departmentId,
                                  final Department department) {

        departmentMap.put(departmentId, department);
        return true;
    }

    @RequestMapping(value = "/department/{departmentId}/employee/", method = RequestMethod.POST)
    public boolean addEmployee(@PathVariable("departmentId") Integer departmentId,
                               final Employee employee) {

        final Department department = departmentMap.get(departmentId);

        if (department ==  null) {
            throw new IllegalArgumentException("Department " + departmentId + " does not exist");
        }

        department.addEmployee(employee);
        return true;
    }

    @RequestMapping(value = "/department/{departmentId}/employee/{employeeId}", method = RequestMethod.GET)
    public Employee getEmployee(@PathVariable("departmentId") Integer departmentId,
                                @PathVariable("employeeId") Long employeeId) {

        final Department department = departmentMap.get(departmentId);

        if (department ==  null) {
            throw new IllegalArgumentException("Department " + departmentId + " does not exist");
        }

        Optional<Employee> employee = department.getEmployeeList().stream().filter(
                employee1 -> employee1.getId() == employeeId).findFirst();

        if (employee.isPresent()){
            return employee.get();
        } else {
            throw new IllegalArgumentException("Employee with id " + employeeId + " Not found ");
        }
    }


    @RequestMapping(value = "/department/{departmentId}/employee/{employeeId}/phoneNumber/",
            method = RequestMethod.POST)
    public boolean addPhoneNumber(@PathVariable("departmentId") Integer departmentId,
                                  @PathVariable("employeeId") Long employeeId,
                                  PhoneNumber phoneNumber) {

        Employee employee = getEmployee(departmentId, employeeId);
        employee.addPhoneNumber(phoneNumber);
        return true;
    }



    @RequestMapping(value = "/department/{departmentId}/employee/{employeeId}/phoneNumber/")
    public List<PhoneNumber> getPhoneNumbers(@PathVariable("departmentId") Integer departmentId,
                                             @PathVariable("employeeId") Long employeeId) {

        Employee employee = getEmployee(departmentId, employeeId);
        return employee.getPhoneNumbers();
    }


    @RequestMapping(value = "/kitchen/{departmentId}/employee/phoneNumber/kitchen/",
            method = RequestMethod.POST)
    public boolean addPhoneNumberKitchenSink(@PathVariable("departmentId") Integer departmentId,
                                  @RequestParam("employeeId") Long employeeId,
                                  @HeaderParam("X-PASS-CODE") String passCode,
                                             PhoneNumber phoneNumber) {

        if ("passcode".equals(passCode)) {
            Employee employee = getEmployee(departmentId, employeeId);
            employee.addPhoneNumber(phoneNumber);
            return true;
        } else {
            return false;
        }
    }



}
Kafka and Cassandra support, training for AWS EC2 Cassandra 3.0 Training