libfstrm-dev 0.3.0-1build1 / usr / include / fstrm.h

/*
 * Copyright (c) 2013-2014 by Farsight Security, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*! \file
 * \mainpage Introduction
 *
 * This is `fstrm`, a C implementation of the Frame Streams data transport
 * protocol.
 *
 * Frame Streams is a light weight, binary clean protocol that allows for the
 * transport of arbitrarily encoded data payload sequences with minimal framing
 * overhead -- just four bytes per data frame. Frame Streams does not specify an
 * encoding format for data frames and can be used with any data serialization
 * format that produces byte sequences, such as [Protocol Buffers], [XML],
 * [JSON], [MessagePack], [YAML], etc. Frame Streams can be used as both a
 * streaming transport over a reliable byte stream socket (TCP sockets, TLS
 * connections, `AF_UNIX` sockets, etc.) for data in motion as well as a file
 * format for data at rest. A "Content Type" header identifies the type of
 * payload being carried over an individual Frame Stream and allows cooperating
 * programs to determine how to interpret a given sequence of data payloads.
 *
 * `fstrm` is an optimized C implementation of Frame Streams that includes a
 * fast, lockless circular queue implementation and exposes library interfaces
 * for setting up a dedicated Frame Streams I/O thread and asynchronously
 * submitting data frames for transport from worker threads. It was originally
 * written to facilitate the addition of high speed binary logging to DNS
 * servers written in C using the [dnstap] log format.
 *
 * This is the API documentation for the `fstrm` library. For the project
 * hosting site, see <https://github.com/farsightsec/fstrm>.
 *
 * \authors Farsight Security, Inc. and the `fstrm` authors.
 *
 * \copyright 2013-2014. Licensed under the terms of the [Apache-2.0] license.
 *
 * [Protocol Buffers]: https://developers.google.com/protocol-buffers/
 * [XML]:              http://www.w3.org/TR/xml11/
 * [JSON]:             http://www.json.org/
 * [MessagePack]:      http://msgpack.org/
 * [YAML]:             http://www.yaml.org/
 * [dnstap]:           http://dnstap.info/
 * [Apache-2.0]:       http://www.apache.org/licenses/LICENSE-2.0
 *
 * \page overview Library overview
 *
 * \section init Initializing the library
 *
 * `fstrm` has no global library state. In most cases, only a single
 * \ref fstrm_iothr library context object will be needed for the entire process,
 * which will implicitly create a background I/O serialization thread. This I/O
 * thread is bound to a particular output writer (for example, an `AF_UNIX`
 * socket) and is fully buffered -- submitted data frames will be accumulated in
 * an output buffer and periodically flushed, minimizing the number of system
 * calls that need to be performed. This frees worker threads from waiting for a
 * write() to complete.
 *
 * `fstrm` abstracts the actual I/O operations needed to read or write a byte
 * stream. File and socket I/O implementations are included in the library, but
 * if necessary `fstrm` can be extended to support new types of byte stream
 * transports. See the \ref fstrm_reader, \ref fstrm_writer, and \ref fstrm_rdwr
 * interfaces for details.
 *
 * The following code example shows the initialization of an `fstrm_iothr`
 * library context object connected to an \ref fstrm_file writer.
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        const char *file_path = "/tmp/output.fs";
        struct fstrm_file_options *fopt;
        struct fstrm_iothr *iothr;
        struct fstrm_writer *writer;

        fopt = fstrm_file_options_init();
        fstrm_file_options_set_file_path(fopt, file_path);

        writer = fstrm_file_writer_init(fopt, NULL);
        if (!writer) {
                fprintf(stderr, "Error: fstrm_file_writer_init() failed.\n");
                exit(EXIT_FAILURE);
        }

        iothr = fstrm_iothr_init(NULL, &writer);
        if (!iothr) {
                fprintf(stderr, "Error: fstrm_iothr_init() failed.\n");
                exit(EXIT_FAILURE);
        }

        fstrm_file_options_destroy(&fopt);
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 *
 * Since the I/O operations are abstracted through the `fstrm_writer` interface,
 * the `writer` variable in the above example could instead have been
 * initialized with a completely different implementation. For example,
 * \ref fstrm_unix_writer objects can be initialized as follows:
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        const char *socket_path = "/tmp/output.sock";
        struct fstrm_writer *writer;
        struct fstrm_unix_writer_options *uwopt;

        uwopt = fstrm_unix_writer_options_init();
        fstrm_unix_writer_options_set_socket_path(uwopt, socket_path);

        writer = fstrm_unix_writer_init(uwopt, NULL);
        if (!writer) {
                fprintf(stderr, "Error: fstrm_unix_writer_init() failed.\n");
                exit(EXIT_FAILURE);
        }
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 *
 * \section queue Getting an input queue
 *
 * After the `fstrm_iothr` object has been created with fstrm_iothr_init(), an
 * input queue handle can be obtained with the fstrm_iothr_get_input_queue()
 * function, which returns an `fstrm_iothr_queue` object. This function is
 * thread-safe and returns a unique queue each time it is called, up to the
 * number of queues specified by fstrm_iothr_options_set_num_input_queues().
 * `fstrm_iothr_queue` objects belong to their parent `fstrm_iothr` object and
 * will be destroyed when the parent `fstrm_iothr` object is destroyed.
 *
 * The following code example shows a single `fstrm_iothr_queue` handle being
 * obtained from an already initialized `fstrm_iothr` library context object.
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        // 'iothr' is a struct fstrm_iothr *

        struct fstrm_iothr_queue *ioq;
        ioq = fstrm_iothr_get_input_queue(iothr);
        if (!ioq) {
                fprintf(stderr, "Error: fstrm_iothr_get_input_queue() failed.\n");
                exit(EXIT_FAILURE);
        }
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 *
 * \section submit Submitting data frames
 *
 * Once the `fstrm_iothr` object has been created and an `fstrm_iothr_queue`
 * handle is available, data frames can be submitted for asynchronous writing
 * using the fstrm_iothr_submit() function. A callback is passed to this
 * function which will be invoked to deallocate the data frame once the I/O
 * thread has completed processing it. In the common case where the data frame
 * is dynamically allocated with `malloc()`, the deallocation callback must call
 * `free()`. fstrm_free_wrapper() is provided as a convenience function which
 * does this and can be specified as the `free_func` parameter to
 * fstrm_iothr_submit().
 *
 * If space is available in the queue, fstrm_iothr_submit() will return
 * #fstrm_res_success, indicating that ownership of the memory allocation for the
 * data frame has passed from the caller to the library. The caller must not
 * reuse or deallocate the memory for the data frame after a successful call to
 * fstrm_iothr_submit().
 *
 * Callers must check the return value of fstrm_iothr_submit(). If this function
 * fails, that is, it returns any result code other than #fstrm_res_success, the
 * caller must deallocate or otherwise dispose of memory allocated for the data
 * frame, in order to avoid leaking memory. fstrm_iothr_submit() can fail with
 * #fstrm_res_again if there is currently no space in the circular queue for an
 * additional frame, in which case a later call to fstrm_iothr_submit() with the
 * same parameters may succeed. However, if fstrm_iothr_submit() fails with
 * #fstrm_res_invalid, then there is a problem with the parameters and a later
 * call will not succeed.
 *
 * The following code example shows data frames containing a short sequence of
 * bytes being created and submitted repeatedly, with appropriate error
 * handling. Note that the data frames in this example intentionally contain
 * embedded unprintable characters, showing that Frame Streams is binary clean.
 * This example follows from the previous examples, where the `iothr` and `ioq`
 * variables have already been initialized.
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        // 'iothr' is a struct fstrm_iothr *
        // 'ioq' is a struct fstrm_queue *

        const unsigned num_frames = 100;
        const uint8_t frame_template[] = {
                'H', 'e', 'l', 'l', 'o', 0x00, 0x01, 0x02, 0x03,
                'W', 'o', 'r', 'l', 'd', 0x04, 0x05, 0x06, 0x07,
        };

        for (unsigned i = 0; i < num_frames; i++) {
                // Allocate a new frame from the template.
                uint8_t *frame = malloc(sizeof(frame_template));
                if (!frame)
                        break;
                memcpy(frame, frame_template, sizeof(frame_template));

                // Submit the frame for writing.
                for (;;) {
                        fstrm_res res;
                        res = fstrm_iothr_submit(iothr, ioq, frame,
                                                 sizeof(frame_template),
                                                 fstrm_free_wrapper, NULL);
                        if (res == fstrm_res_success) {
                                // Frame successfully queued.
                                break;
                        } else if (res == fstrm_res_again) {
                                // Queue is full. Try again in a busy loop.
                                // Alternatively, if loss can be tolerated we
                                // could free the frame here and break out of
                                // the loop.
                                continue;
                        } else {
                                // Permanent failure.
                                free(frame);
                                fputs("fstrm_iothr_submit() failed.\n", stderr);
                                break;
                        }
                }
        }
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 *
 * \section shutdown Shutting down
 *
 * Calling fstrm_iothr_destroy() on the `fstrm_iothr` object will signal the I/O
 * thread to flush any outstanding data frames being written and will deallocate
 * all associated resources. This function is synchronous and does not return
 * until the I/O thread has terminated.
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        // 'iothr' is a struct fstrm_iothr *
        fstrm_iothr_destroy(&iothr);
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 */

#ifndef FSTRM_H
#define FSTRM_H

#ifdef __cplusplus
extern "C" {
#endif

#include <sys/uio.h>
#include <stddef.h>
#include <stdint.h>

/**
 * \defgroup fstrm_res fstrm_res
 *
 * Library result codes.
 * @{
 */

/**
 * Result codes for functions.
 */
typedef enum {
	/** Success. */
	fstrm_res_success,

	/** Failure. */
	fstrm_res_failure,

	/** Resource temporarily unavailable. */
	fstrm_res_again,

	/** Parameters were invalid. */
	fstrm_res_invalid,

	/** The end of a stream has been reached. */
	fstrm_res_stop,
} fstrm_res;

/**@}*/

struct fstrm_control;
struct fstrm_file_options;
struct fstrm_iothr;
struct fstrm_iothr_options;
struct fstrm_iothr_queue;
struct fstrm_rdwr;
struct fstrm_reader_options;
struct fstrm_unix_writer_options;
struct fstrm_writer;
struct fstrm_writer_options;

#include <fstrm/control.h>
#include <fstrm/file.h>
#include <fstrm/iothr.h>
#include <fstrm/rdwr.h>
#include <fstrm/reader.h>
#include <fstrm/unix_writer.h>
#include <fstrm/writer.h>

#ifdef __cplusplus
}
#endif

#endif /* FSTRM_H */