Showing posts with label go. Show all posts
Showing posts with label go. Show all posts

Monday, June 11, 2018

Working with nulls with sql databases in Golang

Go is typed with no generics, so it's a little tricky working with databases where null values abound.

the sql package includes structs such as sql.NullString, but it's confusing for users to get a json response that looks like this, which is how the value is represented internally:

{
 "foo": {
  "Valid": true,
  "Value": 5
 }
}

Below is a working golang HTTP server that handles null strings. It reads and writes to a sqlite3 database (in memory). When rows contain null values, they'll be properly displayed as null.

The trick to making things work is the null.v3 library, which implements the json marshal and unmarshal methods necessary to return a null value when querying the database.

Gist here.

package main

import (
 "database/sql"
 "encoding/json"
 "log"
 "net/http"
 "strconv"

 _ "github.com/mattn/go-sqlite3"
 "gopkg.in/guregu/null.v3"
)

// DB is the database connector
var DB *sql.DB

// Person represents a single row in a database. Using the type null.
type Person struct {
 Name     string      `json:"id"`
 Age      int         `json:"age"`
 NickName null.String `json:"nickname"` // Optional
}

// InsertPerson adds a person to the database
func InsertPerson(p Person) {
 cnx, _ := DB.Prepare(`
          INSERT INTO people (name, age, nickname) VALUES (?, ?, ?)`)
 defer cnx.Close()
  
 log.Printf("Adding person: %v\n", p)
 cnx.Exec(p.Name, p.Age, p.NickName)
}

// GetPeople will retur N number of people from database
func GetPeople(n int) []Person {
 people := make([]Person, 0)
 rows, _ := DB.Query(`SELECT name, age, nickname from people LIMIT ?`, n)
 for rows.Next() {
  p := new(Person)
  rows.Scan(&p.Name, &p.Age, &p.NickName)
  people = append(people, *p)
 }
 return people
}

func addPersonRouter(w http.ResponseWriter, r *http.Request) {
 r.ParseForm()

 age, _ := strconv.Atoi(r.FormValue("age"))

 // Get nickname from the form and create a new null.String. If the string
 // is empty, it will be considered invalid (null) in the database and not
 // empty
 nick := r.FormValue("nickname")
 nickName := null.NewString(
  nick, nick != "")

 p := Person{
  Name:     r.FormValue("name"),
  Age:      age,
  NickName: nickName,
 }

 InsertPerson(p)

 w.WriteHeader(http.StatusCreated)
}

func getPeopleRouter(w http.ResponseWriter, r *http.Request) {
 r.ParseForm()
 limit, _ := strconv.Atoi(r.FormValue("limit"))
 people := GetPeople(limit)

 peopleJSON, _ := json.Marshal(people)
 w.Header().Set("Content-Type", "application/json")
 w.Write(peopleJSON)
}

// CreateTable is a helper function to create the table for the first run
func CreateTable() error {

 createSQL := `
    CREATE TABLE people (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
  name TXT NOT NULL,
  age INTEGER NOT NULL,
  nickname TXT
 );`
  
 statement, err := DB.Prepare(createSQL)
 if err != nil {
  return err
 }

 statement.Exec()
 statement.Close()

 return nil
}

func main() {
 var err error
 DB, err = sql.Open("sqlite3", ":memory:")
 if err != nil {
  log.Fatal(err)
 }

 err = CreateTable()
 if err != nil {
  log.Fatal(err)
 }

 http.HandleFunc("/add", addPersonRouter)
 http.HandleFunc("/list", getPeopleRouter)
 http.ListenAndServe(":8080", nil)
}

Here's some examples of the server when it's running

$ curl -XPOST "localhost:8080/add?name=Joseph&age=25&nickname=Joe"
$ curl -XPOST "localhost:8080/add?name=Stephen&age=33"
$ curl -s 'localhost:8080/list?limit=2' | jq
[
  {
    "id": "Joseph",
    "age": 25,
    "nickname": "Joe"
  },
  {
    "id": "Stephen",
    "age": 33,
    "nickname": null
  }
]

There you go. Now you can read and write valid or invalid values to your database.

Wednesday, November 18, 2015

List all keys in an S3 bucket using golang

Here's a golang snippet that will list all of the keys under your S3 bucket using the official aws-sdk-go:
package main

import (
    "fmt"
    "github.com/aws/aws-sdk-go/aws"
    "github.com/aws/aws-sdk-go/aws/session"
    "github.com/aws/aws-sdk-go/service/s3"
)

func main() {
    svc := s3.New(session.New(), &aws.Config{Region: aws.String("us-east-1")})

    params := &s3.ListObjectsInput{
        Bucket: aws.String("bucket"),
    }

    resp, _ := svc.ListObjects(params)
    for _, key := range resp.Contents {
        fmt.Println(*key.Key)
    }
}

Saturday, February 28, 2015

List running EC2 instances with golang and the aws-sdk-go

Managing multiple AWS accounts can sometimes be tough, even when doing something as simple as matching a private IP address with a hostname.

In the past I used the aws cli tools, but I had to constantly switch both the accounts and the regions when making requests:

# Make it's in the prod-5 account in us-west-1
aws ec2 --profile=prod-5 --region=us-west-1 | grep my_instance

# Okay not that account or region, let's try eu-west-1
aws ec2 --profile=prod-5 --region=eu-west-2 | grep my_instance

Repeat 1x for each account and region

As you can imagine this is extremely time consuming, even when using the CLI tools. I wrote a small tool that will find every single instance you can view using every account available to you (according to your ~/.aws/config). The aggregate results can then be searched.

I choose to use the existing ~/.aws/config file so that it works along side your normal aws cli tools. 

This is a very good use-case for Golang, which has nice concurrency primitives. With five accounts this script (once compiled) will display all of the results in under 1.5 seconds. Not bad.

The result is a space-separated list with some additional values added. It should be easy to find what you're looking for:

$ ./aws_list
i-71930187 prod-manage005 10.0.0.180 t2.micro 207.171.166.22 prod-account
i-71930187 stage-controller7 10.0.0.164 t2.medium 72.21.206.80 staging-account
i-71930187 prod-vpn01 10.0.0.239 m4.large None prod-account
i-71930187 stephen-test-host 10.0.0.216 m3.large 216.58.216.174 test-account
...

The output is a space-separated file and only "running" and "pending" instances are displayed. For a list of filters or instance attributes consult the official documentation.


package main

import (
    "fmt"
    "github.com/awslabs/aws-sdk-go/aws"
    "github.com/awslabs/aws-sdk-go/service/ec2"
    "github.com/vaughan0/go-ini"
    "net/url"
    "os"
    "runtime"
    "strings"
    "sync"
)

func check(e error) {
    if e != nil {
        panic(e)
    }
}

/*
printIds accepts an aws credentials file and a region, and prints out all
instances within the region in a format that's acceptable to us. Currently that
format is like this:

  instance_id name private_ip instance_type public_ip account

Any values that aren't available (such as public ip) will be printed out as
"None"

Because the "name" parameter is user-defined, we'll run QueryEscape on it so that
our output stays as a space-separated line.
*/
func printIds(creds aws.CredentialsProvider, account string, region string, wg *sync.WaitGroup) {
    defer wg.Done()

    svc := ec2.New(&aws.Config{
        Credentials: creds,
        Region:      region,
    })

    // Here we create an input that will filter any instances that aren't either
    // of these two states. This is generally what we want
    params := &ec2.DescribeInstancesInput{
        Filters: []*ec2.Filter{
            &ec2.Filter{
                Name: aws.String("instance-state-name"),
                Values: []*string{
                    aws.String("running"),
                    aws.String("pending"),
                },
            },
        },
    }

    // TODO: Actually care if we can't connect to a host
    resp, _ := svc.DescribeInstances(params)
    // if err != nil {
    //      panic(err)
    // }

    // Loop through the instances. They don't always have a name-tag so set it
    // to None if we can't find anything.
    for idx, _ := range resp.Reservations {
        for _, inst := range resp.Reservations[idx].Instances {

            // We need to see if the Name is one of the tags. It's not always
            // present and not required in Ec2.
            name := "None"
            for _, keys := range inst.Tags {
                if *keys.Key == "Name" {
                    name = url.QueryEscape(*keys.Value)
                }
            }

            important_vals := []*string{
                inst.InstanceID,
                &name,
                inst.PrivateIPAddress,
                inst.InstanceType,
                inst.PublicIPAddress,
                &account,
            }

            // Convert any nil value to a printable string in case it doesn't
            // doesn't exist, which is the case with certain values
            output_vals := []string{}
            for _, val := range important_vals {
                if val != nil {
                    output_vals = append(output_vals, *val)
                } else {
                    output_vals = append(output_vals, "None")
                }
            }
            // The values that we care about, in the order we want to print them
            fmt.Println(strings.Join(output_vals, " "))
        }
    }
}

func main() {
    // Go for it!
    runtime.GOMAXPROCS(runtime.NumCPU())

    // Make sure the config file exists
    config := os.Getenv("HOME") + "/.aws/config"
    if _, err := os.Stat(config); os.IsNotExist(err) {
        fmt.Println("No config file found at: %s", config)
        os.Exit(1)
    }

    var wg sync.WaitGroup

    file, err := ini.LoadFile(config)
    check(err)

    for key, values := range file {
        profile := strings.Fields(key)

        // Don't print the default or non-standard profiles
        if len(profile) != 2 {
            continue
        }

        // Where to find this host. The account isn't necessary for the creds
        // but it's something we expose to users when we print
        account := profile[1]
        key := values["aws_access_key_id"]
        pass := values["aws_secret_access_key"]
        creds := aws.Creds(key, pass, "")

        // Gather a list of all available AWS regions. Even though we're gathering
        // all regions, we still must use a region here for api calls.
        svc := ec2.New(&aws.Config{
            Credentials: creds,
            Region:      "us-west-1",
        })

        // Iterate over every single stinking region to get a list of available
        // ec2 instances
        regions, err := svc.DescribeRegions(&ec2.DescribeRegionsInput{})
        check(err)
        for _, region := range regions.Regions {
            wg.Add(1)
            go printIds(creds, account, *region.RegionName, &wg)
        }
    }

    // Allow the goroutines to finish printing
    wg.Wait()
}