Add rest api doc to ReadTheDoc in rst format.

docs/source/advanced/index.rst

@@ -17,6 +17,7 @@ Advanced Topics
    networks/index.rst
    ports/xcat_ports.rst
    raid/index.rst
+   restapi/index.rst
    security/index.rst
    softlayer/index.rst
    switches/index.rst

docs/source/advanced/restapi/index.rst

@@ -0,0 +1,14 @@
+Rest API
+========
+
+xCAT provides REST API (also called a web services API) that is currently implemented as a front end (translation layer) to xcatd. This provides programmatic access to xCAT from any language. This document describes how to set it up and use it.
+
+**NOTE:** This doc is working with xCAT 2.8.4 and later.
+
+.. toctree::
+   :maxdepth: 2
+
+   restapi_setup/index
+   restapi_usage/index
+   restapi_resource/index
+   

docs/source/advanced/restapi/restapi_resource/index.rst

@@ -0,0 +1,7 @@
+xCAT Rest API Resources
+=======================
+
+.. toctree::
+   :maxdepth: 2
+
+   restapi_reference

docs/source/advanced/restapi/restapi_setup/index.rst

@@ -0,0 +1,9 @@
+Set Up Web Service for Rest API
+===============================
+
+The following steps describe how to setup the Web Service for xCAT management node to offer the Rest API service.
+
+.. toctree::
+   :maxdepth: 2
+
+   restapi_setup

docs/source/advanced/restapi/restapi_setup/restapi_setup.rst

@@ -0,0 +1,142 @@
+Enable the HTTPS service for REST API 
+=====================================
+
+To improve the security between the REST API client and server, enabling the HTTPS service on the xCAT management is recommended. And the REST API client should use the 'https' to access web server instead of the 'http'. 
+
+* **[RHEL6 (x86_64/ppc64) and RHEL5 (x86_64)]** ::
+
+    yum install mod_ssl
+    service httpd restart
+    yum install perl-JSON
+
+* **[RHEL5 (ppc64)]**
+
+  Uninstall httpd.ppc64 and install httpd.ppc: ::
+
+    rpm -e --nodeps httpd.ppc64 
+    rpm -i httpd.ppc mod_ssl.ppc
+
+* **[SLES10/11 (x86_64/ppc64)]** ::
+
+    a2enmod ssl
+    a2enflag SSL
+    /usr/bin/gensslcert
+    cp /etc/apache2/vhosts.d/vhost-ssl.template /etc/apache2/vhosts.d/vhost-ssl.conf
+    Insert line 'NameVirtualHost *:443' before the line '## SSL Virtual Host Context'
+    /etc/init.d/apache2 restart
+    zypper install perl-JSON
+
+* **[Ubuntu]** ::
+
+    sudo a2enmod ssl
+    ln -s ../sites-available/default-ssl.conf  /etc/apache2/sites-enabled/ssl.conf
+    sudo service apache2 restart
+    
+    # verify it is loaded:
+
+    sudo apache2ctl -t -D DUMP_MODULES | grep ssl
+    apt-get install libjson-perl
+
+Enable the Certificate of HTTPs Server (Optional)
+=================================================
+
+Enabling the certificate functionality of https server is useful for the Rest API client to authenticate the server. 
+
+The certificate for xcatd has already been generated when installing xCAT, it can be reused by the https server. To enable the server certificate authentication, the hostname of xCAT MN must be a fully qualified domain name (FQDN). The REST API client also must use this FQDN when accessing the https server. If the hostname of the xCAT MN is not a FQDN, you need to change the hostname first. 
+
+Typically the hostname of the xCAT MN is initially set to the NIC which faces to the cluster (usually an internal/private NIC). If you want to enable the REST API for public client, please set the hostname of xCAT MN to one of the public NIC. 
+
+To change the hostname, edit /etc/sysconfig/network (RHEL) or /etc/HOSTNAME (SLES) and run:  ::
+
+    hostname <newFQDN>
+
+After change the hostname, run the xcat command ``xcatconfig`` to generate a new server certificate base on the correct hostname: ::
+
+    xcatconfig -c
+
+``Notes:`` If you had previously generated a certificate for non-root userids to use xCAT, you must regenerate them using: /opt/xcat/share/xcat/scripts/setup-local-client.sh <username>
+
+The steps to configure the certificate for https server: ::
+
+    export sslcfgfile=/etc/httpd/conf.d/ssl.conf              # rhel
+    export sslcfgfile=/etc/apache2/vhosts.d/vhost-ssl.conf    # sles
+    export sslcfgfile=/etc/apache2/sites-enabled/ssl.conf     # ubuntu
+
+    sed -i 's/^\(\s*\)SSLCertificateFile.*$/\1SSLCertificateFile \/etc\/xcat\/cert\/server-cred.pem/' $sslcfgfile    
+    sed -i 's/^\(\s*\)SSLCertificateKeyFile.*$/\1SSLCertificateKeyFile \/etc\/xcat\/cert\/server-cred.pem/' $sslcfgfile
+        
+    service httpd restart        # rhel
+    service apache2 restart      # sles/ubuntu
+
+The REST API client needs to download the xCAT certificate CA from the xCAT http server to authenticate the certificate of the server. ::
+
+    cd /root
+    wget http://<xcat MN>/install/postscripts/ca/ca-cert.pem
+
+When accessing the REST API, the certificate CA must be specified and the FQDN of the https server must be used. For example: ::
+
+    curl -X GET --cacert /root/ca-cert.pem 'https://<FQDN of xCAT MN>/xcatws/nodes?userName=root& userPW=cluster'
+
+Extend the Timeout of Web Server
+================================
+
+Some operations like 'create osimage' (copycds) need a long time (longer than 3 minutes sometimes) to complete. It would fail with a ``timeout error`` (504 Gateway Time-out) if the timeout setting in the web server is not extended: ::
+
+    For [RHEL]
+        sed -i 's/^Timeout.*/Timeout 600/' /etc/httpd/conf/httpd.conf
+        service htttd restart
+    For [SLES]
+        echo "Timeout 600" >> /etc/apache2/httpd.conf
+        service apache2 restart
+
+Set Up an Account for Web Service Access
+========================================
+
+User needs a username and password to access the REST API. When the REST API request is passed to xcatd, the username and password will be verified base on the :doc:`xCAT passwd Table </guides/admin-guides/references/man5/passwd.5>`, and then xcatd will look in the `xCAT policy Table </guides/admin-guides/references/man5/policy.5>` to see if the user is allowed to perform the requested operation. 
+
+The account which key is **xcat** will be used for the REST API authentication. The username and password should be passed in as the attirbutes of URL: 
+    userName: Pass the username of the account 
+    userPW:   Pass the password of the account (xCAT 2.10)
+    password: Pass the password of the account (xCAT earlier than 2.10)
+
+You can use the root userid for your API calls, but we recommend you create a new userid (for example wsuser) for the API calls and give it the specific privileges you want it to have.
+
+Use root Account
+----------------
+
+The certificate and ssh keys for **root** account has been created during the install of xCAT. The public ssh key also has been uploaded to computer node so that xCAT MN can ssh to CN without password. Then the only thing needs to do is to add the password for the **root** in the passwd table. ::
+
+    tabch key=xcat,username=root passwd.password=<root-pw>
+
+Use non-root Account
+--------------------
+
+Create new user and setup the password and policy rules. ::
+
+    useradd wsuser
+    passwd wsuser     # set the password
+    tabch key=xcat,username=wsuser passwd.password=cluster
+    mkdef -t policy 6 name=wsuser rule=allow
+
+``Note:`` in the tabch command above you can put the salted password (from /etc/shadow) in the xCAT passwd table instead of the clear text password, if you prefer. 
+
+Create the SSL certificate under that user's home directory so that user can be authenticated to xCAT. This is done by running the following command on the Management node as root: ::
+
+    /opt/xcat/share/xcat/scripts/setup-local-client.sh <username>
+
+When running this command you'll see SSL certificates created. Enter "y" where prompted and take the defaults. 
+
+To enable the POST method of resources like nodeshell,nodecopy,updating,filesyncing for the non-root user, you need to enable the ssh communication between xCAT MN and CN without password. Log in as <username> and run following command: ::
+
+    xdsh <noderange> -K
+
+Run a test request to see if everything is working: ::
+
+    curl -X GET --cacert /root/ca-cert.pem 'https://<xcat-mn-host>/xcatws/nodes?userName=<user>&userPW=<password>'
+
+or if you did not set up the certificate: ::
+
+    curl -X GET -k 'https://<xcat-mn-host>/xcatws/nodes?userName=<user>&userPW=<password>'
+
+You should see some output that includes your list of nodes. 
+

docs/source/advanced/restapi/restapi_usage/index.rst

@@ -0,0 +1,7 @@
+How to Use xCAT Rest API
+========================
+
+.. toctree::
+   :maxdepth: 2
+
+   restapi_usage

docs/source/advanced/restapi/restapi_usage/restapi_usage.rst

@@ -0,0 +1,265 @@
+The Resource Categories
+=======================
+
+The API lets you query, change, and manage the resources in following categories:
+
+  * Token Resources
+  * Node Resources
+  * Osimage Resources
+  * Network Resources
+  * Policy Resources
+  * Group Resources
+  * Global Configuration Resources
+  * Service Resources
+  * Table Resources
+
+
+The Authentication Methods for REST API
+=======================================
+
+xCAT REST API supports two ways to authenticate the access user: user account (username + password) and access token (acquired by username + password).
+
+User Account
+------------
+
+Follow the steps in `WEB Service Setup </advanced/restapi/restapi_setup/index>`, you have created an account for yourself. Use this pair of username and password, you can access the https server.
+
+The general format of the URL used in the REST API call is: ::
+
+    https://<FQDN of xCAT MN>/xcatws/<resource>?userName=<user>&userPW=<pw>&<parameters>
+
+where:
+
+  * **FQDN of xCAT MN**: the hostname of the xCAT management node. It also can be the IP of xCAT MN if you don't want to enable the web server certificate
+  * **resource**: one of the xCAT resources listed above
+  * **user**: the userid that the operation should be run on behalf of. See the previous section on how to add/authorize a userid.
+  * **pw**: the password of the userid (can be the salted version from /etc/shadow)
+
+Example: ::
+
+    curl -X GET --cacert /root/ca-cert.pem 'https://<FQDN of xCAT MN>/xcatws/nodes?userName=root&userPW=cluster'
+
+Access Token
+------------
+
+xCAT also supports to use the Access Token to replace the using of username+password in every access. Before the access to any resource, you need get a token first with your account (username+password) ::
+
+    # curl -X POST --cacert /root/ca-cert.pem \
+        'https://<FQDN of xCAT MN>/xcatws/tokens?pretty=1' -H Content-Type:application/json --data \
+        '{"userName":"root","userPW":"cluster"}'
+     {
+        "token":{
+          "id":"5cabd675-bc2e-4318-b1d6-831fd1f32f97",
+          "expire":"2014-3-10 9:56:12"
+        }
+     }
+
+Then in the subsequent REST API access, the token can be used to replace the user account (username+password)  ::
+
+    curl -X GET --cacert /root/ca-cert.pem -H X-Auth-Token:5cabd675-bc2e-4318-b1d6-831fd1f32f97 'https://<FQDN of xCAT MN>/xcatws/<resource>?<parameters>
+
+The validity of token of 24 hours. If an used token has been expired, you will get a 'Authentication failure' error. Then you need reacquire a token with your account.
+
+
+The Common Parameters for Resource URI
+======================================
+
+xCAT REST API supports to use several common parameters in the resource URI to enable specific output:
+
+* **pretty=1** \- It is used to format the json output for easier viewing on the screen. ::
+
+    https://<xCAT MN>/xcatws/nodes?pretty=1
+
+* **debug=1** \- It is used to display more debug messages for a REST API request. ::
+
+    https://<xCAT MN>/xcatws/nodes?debug=1
+
+* **xcoll=1** \- It is used to specify that the output should be grouped with the values of objects. ::
+
+    GET https://<xCAT MN>/xcatws/nodes/node1,node2,node3/power?xcoll=1
+     {
+       "node2":{
+         "power":"off"
+       },
+       "node1,node3":{
+         "power":"on"
+       }
+     }
+
+``Note:`` All the above parameters can be used in mixed like following: ::
+
+    https://<xCAT MN>/xcatws/nodes?pretty=1&debug=1
+
+
+The Output of REST API request
+==============================
+
+xCAT REST API only supports the [JSON](http://www.json.org/) formatted output.
+
+When an Error occurs during the operation
+-----------------------------------------
+
+(i.e. there's error/errorcode in the output of xcat xml response):
+
+When error happens, for all the GET/PUT/POST/DELETE methods, the output will only includes 'error' and 'errorcode' properties: ::
+
+    {
+       error:[
+           msg1,
+           msg2,
+           ...
+       ],
+       errorcode:error_number
+    }
+
+When NO Error occurs during the operation
+-----------------------------------------
+
+(i.e. there's no error/errorcode in the output of xcat xml response):
+
+For the GET method
+``````````````````
+
+If the output can be grouped by the object (resource) name, and the information being returned are attributes of the object, then use the object name as the hash key and make the value be a hash of its attributes/values: ::
+
+    {
+      object1: {
+         a1: v1,
+         a2: v2,
+         ...
+      },
+      object2: {
+         a1: v1,
+         a2: v2,
+         ...
+      },
+    }
+
+If the output can be grouped by the object (resource) name, but the information being returned is **not** attributes of the object, then use the object name as the hash key and make the value be an array of strings: ::
+
+    {
+      object1: [
+         msg1,
+         msg2,
+         ...
+      ],
+      object2: [
+         msg1,
+         msg2
+         ...
+      ],
+    }
+
+An example of this case is the output of reventlog: ::
+
+    {
+      "node1": [
+         "09/07/2013 10:05:02 Event Logging Disabled, Log Area Reset/Cleared (SEL Fullness)",
+         ...
+      ],
+    }
+
+If the output is not object related, put all the output in a list (array): ::
+
+    [
+       msg1,
+       msg2,
+       ...
+    ]
+
+For the PUT/DELETE methods
+``````````````````````````
+
+There will be no output for operations that succeed. (We made this decision because the output for them is always not formatted, and no program will read it if xcat indicates the operation has succeeded.)
+
+For POST methods
+````````````````
+
+Since POST methods can either be creates or general actions, there is not as much consistency. In the case of a create, the rule is the same as PUT/DELETE (no output if successful). For actions that have output that matters (e.g. nodeshell, filesyncing, sw, postscript), the rules are like the GET method.
+
+
+Testing the API
+===============
+
+Normally you will make REST API calls from your code. You can use any language that has REST API bindings (most modern languages do).
+
+An Example of How to Use xCAT REST API from PERL
+------------------------------------------------
+
+Refer to the file /opt/xcat/ws/xcatws-test.pl: ::
+
+    ./xcatws-test.pl -m GET -u "https://127.0.0.1/xcatws/nodes?userName=root&userPW=cluster"
+
+An Example Script of How to Use curl to Test Your xCAT REST API Service
+-----------------------------------------------------------------------
+
+It can be used as an initial script to make your xCAT REST API script to access and control xCAT resources. From the output message, you also could get the idea of how to access xCAT resources. ::
+
+    /opt/xcat/ws/xcatws-test.sh
+    ./xcatws-test.sh -u root -p cluster
+    ./xcatws-test.sh -u root -p cluster -h <FQDN of xCAT MN>
+    ./xcatws-test.sh -u root -p cluster -h <FQDN of xCAT MN> -c
+    ./xcatws-test.sh -u root -p cluster -h <FQDN of xCAT MN> -t
+    ./xcatws-test.sh -u root -p cluster -h <FQDN of xCAT MN> -c -t
+
+But for exploration and experimentation, you can make API calls from your browser or using the **curl** command.
+
+To make an API call from your browser, use the desired URL from this document. To simplify the test step, all the examples for the resources uses 'curl -k' to use insecure http connection and use the 'username+password' to authenticate the user. ::
+
+    curl -X GET -k 'https://myserver/xcatws/nodes?userName=xxx&userPW=xxx&pretty=1'
+
+Examples of making an API call using curl
+-----------------------------------------
+
+* **To query resources:** ::
+
+    curl -X GET -k 'https://xcatmnhost/xcatws/nodes?userName=xxx&userPW=xxx&pretty=1'
+
+* **To change attributes of resources:** ::
+
+    curl -X PUT -k 'https://xcatmnhost/xcatws/nodes/{noderange}?userName=xxx&userPW=xxx' \
+       -H Content-Type:application/json --data '{"room":"hi","unit":"7"}'
+
+* **To run an operation on a resource:** ::
+
+    curl -X POST -k 'https://xcatmnhost/xcatws/nodes/{noderange}?userName=xxx&userPW=xxx' \
+       -H Content-Type:application/json --data '{"groups":"wstest"}'
+
+* **To delete a resource:** ::
+
+    curl -X DELETE -k 'https://xcatmnhost/xcatws/nodes/{noderange}?userName=xxx&userPW=xxx'
+
+
+Web Service Status Codes
+========================
+
+Here are the HTTP defined status codes that the Web Service can return:
+
+  * 401 Unauthorized
+  * 403 Forbidden
+  * 404 Not Found
+  * 405 Method Not Allowed
+  * 406 Not Acceptable
+  * 408 Request Timeout
+  * 417 Expectation Failed
+  * 418 I'm a teapot
+  * 503 Service Unavailable
+  * 200 OK
+  * 201 Created
+
+References
+==========
+
+  * REST: http://en.wikipedia.org/wiki/Representational_State_Transfer
+  * REST: http://rest.elkstein.org/2008/02/what-is-rest.html
+  * HTTP Status codes: http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
+  * HTTP Request Methods: http://tools.ietf.org/html/rfc2616#section-9.1
+  * HTTP Request Tool: http://soft-net.net/SendHTTPTool.aspx (haven't tried it yet)
+  * HTTP PATCH: http://tools.ietf.org/html/rfc5789
+  * HTTP BASIC Security: http://httpd.apache.org/docs/2.2/mod/mod_auth_basic.html
+  * Asynchronous Rest: http://www.infoq.com/news/2009/07/AsynchronousRest
+  * General JSON: http://www.json.org/
+  * JSON wrapping: http://search.cpan.org/~makamaka/JSON-2.27/lib/JSON.pm
+  * Apache CGI: http://httpd.apache.org/docs/2.2/howto/cgi.html
+  * Perl CGI: http://perldoc.perl.org/CGI.html
+

xCAT-server/xCAT-wsapi/genrestapidoc.pm

@@ -70,11 +70,134 @@ my %formathdl = (
     text => \&outtext,
     wiki => \&outwiki,
     mediawiki => \&outmediawiki,
+    rst => \&make_rst_format,
 );
 
 
 my @errmsg;
 
+#--------------------------------------------------------------------------------
+
+=head3   make_rst_format
+
+    Descriptions:
+        This subroutine is used to generate restapi doc in rst format
+    Arguments:
+        param1: The first parameter
+        param2: The second parameter
+        param3: The third parameter
+    Returns:
+        0 - success
+        0 - fail
+=cut
+
+#--------------------------------------------------------------------------------
+sub make_rst_format {
+    my $def = shift;
+    my $opt = shift;
+    my $res = shift;
+
+    if ($res) {
+        if (defined ($res->{'desc'})) {
+            # add \ for [ and ]
+            #$res->{'desc'} =~ s/\[/\\\[/;
+            #$res->{'desc'} =~ s/\]/\\\]/;
+            print "$res->{'desc'}\n";
+            print "-" x length($res->{'desc'}) . "\n\n";
+        }
+        foreach (1..10) {
+            if (defined ($res->{'desc'.$_})) {
+                print $res->{'desc'.$_}."\n\n";
+            }
+        }
+    }
+
+    my $postfix = "?userName=root&userPW=cluster&pretty=1";
+
+    if (defined ($def->{desc})) {
+        print "$opt - $def->{desc}\n";
+        print "`" x length("$opt - $def->{desc}") . "\n\n";
+    }
+    foreach (1..10) {
+        if (defined ($def->{'desc'.$_})) {
+            print $def->{'desc'.$_}."\n\n";
+        }
+    }
+
+    if (defined ($def->{cmd})) {
+        my $manpath = search_manpage($def->{cmd});
+
+        if ($manpath) {
+            print "Refer to the man page: :doc:`$def->{cmd} <$manpath>`\n\n";
+        } else {
+            print "Refer to the man page of ".$def->{cmd}." command.\n\n";
+        }
+
+    }
+
+    if (defined ($def->{usage})) {
+        #$def->{usage} =~ s/\[/\\\[/;
+        #$def->{usage} =~ s/\]/\\\]/;
+        my @parts = split ('\|', $def->{usage});
+        if ($parts[1]) {
+            print "**Parameters:**\n\n* $parts[1]\n\n";
+        }
+        if ($parts[2]) {
+            print "**Returns:**\n\n* $parts[2]\n\n";
+        }
+    }
+
+    my @example_array = ();
+    if (defined($def->{example})) {
+        push @example_array, $def->{example};
+    }
+    foreach (1..10) {
+        if (defined($def->{'example'.$_})) {
+            push @example_array, $def->{'example'.$_};
+        }
+    }
+
+    if (@example_array) {
+        my $exampleno = "";
+        if ($#example_array > 0) {
+            $exampleno = 1;
+        }
+        foreach my $line (@example_array) {
+            my @parts = split ('\|', $line);
+            print "**Example$exampleno:** \n\n";
+            if ($#example_array > 0) {
+                $exampleno++;
+            }
+
+            if ($parts[1]) {
+                print "$parts[1] :: \n\n";
+            } else {
+                push @errmsg, "Error format for:[".$def->{desc}."]\n";
+            }
+
+            if ($parts[2] && $parts[3] && ($parts[4] || $opt ne "GET")) {
+                my ($uri, $data);
+                if ($parts[3] =~ /\s+/) {
+                    ($uri, $data) = split(/ /, $parts[3]);
+                    print "\n    #curl -X $parts[2] -k \'https://127.0.0.1/xcatws$uri$postfix\' -H Content-Type:application/json --data \'$data\'\n";
+                } else {
+                    print "\n    #curl -X $parts[2] -k \'https://127.0.0.1/xcatws$parts[3]$postfix\'\n";
+                }
+
+                if ($parts[4]) {
+                    $parts[4] =~ s/\n/\n    /g;
+                    print "    $parts[4]\n\n";
+                }
+            } else {
+                push @errmsg, "Error format for:[".$def->{desc}."]\n";
+            }
+        }
+    } else {
+        push @errmsg, "Error format for:[".$def->{desc}."]\n";
+    }
+}
+
+
 sub outtext {
     my $def = shift;
     my $opt = shift;
@@ -354,11 +477,12 @@ sub outmediawiki {
 sub search_manpage {
     my $cmd = shift;
 
-    if (-d "/opt/xcat/share/man") {
-        my $run = "cd /opt/xcat/share/man; find . | grep \'$cmd\\.\'";
+    if (-d "../../docs/source/guides/admin-guides/references/") {
+        my $run = "find ../../docs/source/guides/admin-guides/references/ | grep \'$cmd\\.\'";
         my @output = `$run`;
         if (@output) {
-            $output[0] =~ s/^\.//;
+            $output[0] =~ s/..\/..\/docs\/source//;
+            $output[0] =~ s/\.rst$//;
             chomp($output[0]);
             return $output[0];
         }
@@ -396,6 +520,10 @@ sub gendoc {
             } elsif ($format eq "mediawiki") {
                 print "==".$group->{'header'}."==\n";
                 print $group->{'desc'}."\n";
+            } elsif ($format eq "rst") {
+                print $group->{'header'} . "\n";
+                print "=" x length($group->{'header'}) . "\n\n";
+                print $group->{'desc'}."\n\n";
             }
             foreach my $res (@{$group->{'resources'}}) {
                 if (defined ($URIdef->{$groupname}->{$res})) {

xCAT-server/xCAT-wsapi/xcatws.cgi

@@ -1284,7 +1284,7 @@ my $xmlinstalled;    # Global var to speicfy whether the xml modules have been l
 # This script also support to generate the rest api doc automatically.
 # Following part of code will not be run when this script is called by http server
 my $dbgdata;
-sub dbgusage { print "Usage:\n    $0 -h\n    $0 -g [wiki] (generate document)\n    $0 {GET|PUT|POST|DELETE} URI user:password \'{data}\'\n"; }
+sub dbgusage { print "Usage:\n    $0 -h\n    $0 -g rst > ../../docs/source/advanced/restapi/restapi_resource/restapi_reference.rst (generate document)\n    $0 {GET|PUT|POST|DELETE} URI user:password \'{data}\'\n"; }
 
 if ($ARGV[0] eq "-h") {
     dbgusage();