SSL/HTTPS client in C

Question

I've written a simple SSL/HTTPS client in C using some example code I found, when I use it to send a GET request to an https server I get an unusual response, this is the response from stackoverflow.com:

HTTP/1.1 200 OK Cache-Control: public, no-cache="Set-Cookie", max-age=36 Content-Type: text/html; charset=utf-8 Expires: Sat, 03 Jan 2015 16:54:57 GMT Last-Modified: Sat, 03 Jan 2015 16:53:57 GMT Vary: * X-Frame-Options: SAMEORIGIN Set-Cookie: prov=407726d8-1493-4ebd-8657-8958be5b2683; domain=.stackoverflow.com; expires=Fri, 01-Jan-2055 00:00:00 GMT; path=/; HttpOnly Date: Sat, 03 Jan 2015 16:54:20 GMT Content-Length: 239129

<title>Stack Overflow</title>
<link rel="shortcut icon" href="//cdn.sstatic.net/stackoverflow/img/favicon.ico?v=038622610830">
<link rel="apple-touch-icon image_src" href="//cdn.sstatic.net/stackoverflow/img/apple-touch-icon.png?v=fd7230a85918">
<link rel="search" type="application/opensearchdescription+xml" title="Stack Overflow" href="/opensearch.xml">
<meta name="twitter:card" content="summary">
<meta name="twitter:domain" content="stackoverflow.com"/>
<meta property="og:type" content="website" />
<meta property="og:image" itemprop="image primaryImageOfPage" content="http://cdn.sstatic.net/stackoverflow/img/[email protected]?v=fde65a5a78c6"

/>

When I use the openssl command line tool to perform the same operation I get a normal response containing the index page. I've tried changing some of the code and followed different tutorials but nothing seems to work. How do I get the program to return the index page instead of the response I currently get?, here's the source code for the program:

#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <openssl/bio.h>
#include <openssl/ssl.h>
#include <openssl/err.h>

/**
 * Simple log function
 */
void slog(char* message) {
    fprintf(stdout, message);
}

/**
 * Print SSL error details
 */
void print_ssl_error(char* message, FILE* out) {

    fprintf(out, message);
    fprintf(out, "Error: %s\n", ERR_reason_error_string(ERR_get_error()));
    fprintf(out, "%s\n", ERR_error_string(ERR_get_error(), NULL));
    ERR_print_errors_fp(out);
}

/**
 * Print SSL error details with inserted content
 */
void print_ssl_error_2(char* message, char* content, FILE* out) {

    fprintf(out, message, content);
    fprintf(out, "Error: %s\n", ERR_reason_error_string(ERR_get_error()));
    fprintf(out, "%s\n", ERR_error_string(ERR_get_error(), NULL));
    ERR_print_errors_fp(out);
}

/**
 * Initialise OpenSSL
 */
void init_openssl() {

    /* call the standard SSL init functions */
    SSL_load_error_strings();
    SSL_library_init();
    ERR_load_BIO_strings();
    OpenSSL_add_all_algorithms();

    /* seed the random number system - only really nessecary for systems without '/dev/random' */
    /* RAND_add(?,?,?); need to work out a cryptographically significant way of generating the seed */
}


/**
 * Connect to a host using an encrypted stream
 */
BIO* connect_encrypted(char* host_and_port, char* store_path, SSL_CTX** ctx, SSL** ssl) {

    BIO* bio = NULL;
    int r = 0;

    /* Set up the SSL pointers */
    *ctx = SSL_CTX_new(TLSv1_client_method());
    *ssl = NULL;
        r = SSL_CTX_load_verify_locations(*ctx, store_path, NULL);

    if (r == 0) {

        print_ssl_error_2("Unable to load the trust store from %s.\n", store_path, stdout);
        return NULL;
    }

    /* Setting up the BIO SSL object */
    bio = BIO_new_ssl_connect(*ctx);
    BIO_get_ssl(bio, ssl);
    if (!(*ssl)) {

        print_ssl_error("Unable to allocate SSL pointer.\n", stdout);
        return NULL;
    }
    SSL_set_mode(*ssl, SSL_MODE_AUTO_RETRY);

    /* Attempt to connect */
    BIO_set_conn_hostname(bio, host_and_port);

    /* Verify the connection opened and perform the handshake */
    if (BIO_do_connect(bio) < 1) {

        print_ssl_error_2("Unable to connect BIO.%s\n", host_and_port, stdout);
        return NULL;
    }

    if (SSL_get_verify_result(*ssl) != X509_V_OK) {

        print_ssl_error("Unable to verify connection result.\n", stdout);
    }

    return bio;
}

/**
 * Read a from a stream and handle restarts if nessecary
 */
ssize_t read_from_stream(BIO* bio, char* buffer, ssize_t length) {

    ssize_t r = -1;

    while (r < 0) {

        r = BIO_read(bio, buffer, length);
        if (r == 0) {

            print_ssl_error("Reached the end of the data stream.\n", stdout);
            continue;

        } else if (r < 0) {

            if (!BIO_should_retry(bio)) {

                print_ssl_error("BIO_read should retry test failed.\n", stdout);
                continue;
            }

            /* It would be prudent to check the reason for the retry and handle
             * it appropriately here */
        }

    };

    return r;
}

/**
 * Write to a stream and handle restarts if nessecary
 */
int write_to_stream(BIO* bio, char* buffer, ssize_t length) {

    ssize_t r = -1;

    while (r < 0) {

        r = BIO_write(bio, buffer, length);
        if (r <= 0) {

            if (!BIO_should_retry(bio)) {

                print_ssl_error("BIO_read should retry test failed.\n", stdout);
                continue;
            }

            /* It would be prudent to check the reason for the retry and handle
             * it appropriately here */
        }

    }

    return r;
}

/**
 * Main SSL demonstration code entry point
 */
int main() {

    char* host_and_port = "stackoverflow.com:443"; 
    char* server_request = "GET / HTTP/1.1\r\nHost: stackoverflow.com\r\n\r\n"; 
    char* store_path = "mycert.pem"; 
    char buffer[4096];
    buffer[0] = 0;

    BIO* bio;
    SSL_CTX* ctx = NULL;
    SSL* ssl = NULL;

    /* initilise the OpenSSL library */
    init_openssl();

        if ((bio = connect_encrypted(host_and_port, store_path, &ctx, &ssl)) == NULL)
            return (EXIT_FAILURE);


    write_to_stream(bio, server_request, strlen(server_request));
    read_from_stream(bio, buffer, 4096);
    printf("%s\r\n", buffer);

     /* clean up the SSL context resources for the encrypted link */
        SSL_CTX_free(ctx);

    return (EXIT_SUCCESS);
}

Solution 1

You call read_from_stream to read at most 4096 bytes, but the answer may be much longer than this. You must retry to read until the call returns 0. You msut also clean the buffer before each read. Like this:

int l;
bzero(buffer,4096);                            // clean the buffer
while (l=read_from_stream(bio,buffer,4096)) { // try to read 4096 bytes from stream to buffer
  printf("%s",buffer);                         // write exactly what was read...
  bzero(buffer,4096);                          // clean the buffer
}

Be careful that the server can send you ASCII nul bytes (rare in HTML pages, but possible for another kind of data)... This code doesn't take this into account.

Normally you have to decode headers, and decode the Content-Length: one. It is intended to give you the number of bytes of data to read after HTTP headers (in your example it is 239129).

Solution 2

Apart from the fact that you are reading only a fixed number of bytes (as mentioned in the other response) you are doing a HTTP/1.1 request. This means that the length of the answer might be specified by a Content-Length header, by Transfer-Encoding: chunked or by end-of-file. And since HTTP/1.1 allows by default multiple requests over a single connection (keep-alive) the simple approach of reading until the end might cause your connection to stall because the server does not close the connection and instead waits for a new request.

If you would issue instead a HTTP/1.0 request you would not have to deal with all these problems, i.e. no chunked mode and no keep-alive by default. Then you could simply read until you get no more data.