diff options
| author | Darshit Shah <darnir@gmail.com> | 2014-09-14 16:57:15 +0530 |
|---|---|---|
| committer | Darshit Shah <darnir@gmail.com> | 2014-09-14 16:57:15 +0530 |
| commit | af702340a1f3b462d0224968602c2b2974f218d6 (patch) | |
| tree | a1d352f6d30505ecc856d5b7561d4dc001b3a891 | |
| parent | 6140b1b6e3831a35fd2cf4536fda741a4dcc9019 (diff) | |
| parent | f8e9a64ec7445d22f67f1817043d5807c4766556 (diff) | |
| download | wget-parallel-wget.tar.gz | |
Merge remote-tracking branch 'origin/master' into parallel-wgetparallel-wget
Conflicts:
src/http.c
testenv/Makefile.am
testenv/Test--https.py
testenv/Test--spider-r.py
testenv/Test-Content-disposition-2.py
testenv/Test-Content-disposition.py
testenv/Test-Head.py
testenv/Test-O.py
testenv/Test-Parallel-Proto.py
testenv/Test-Post.py
testenv/Test-Proto.py
testenv/Test-auth-basic-fail.py
testenv/Test-auth-basic.py
testenv/Test-auth-both.py
testenv/Test-auth-digest.py
testenv/Test-auth-no-challenge-url.py
testenv/Test-auth-no-challenge.py
testenv/Test-auth-retcode.py
testenv/Test-auth-with-content-disposition.py
testenv/Test-c-full.py
testenv/Test-cookie-401.py
testenv/Test-cookie-domain-mismatch.py
testenv/Test-cookie-expires.py
testenv/Test-cookie.py
testenv/conf/__init__.py
testenv/conf/authentication.py
testenv/conf/expect_header.py
testenv/conf/expected_files.py
testenv/conf/expected_ret_code.py
testenv/conf/files_crawled.py
testenv/conf/hook_sample.py
testenv/conf/local_files.py
testenv/conf/reject_header.py
testenv/conf/response.py
testenv/conf/send_header.py
testenv/conf/server_files.py
testenv/conf/urls.py
testenv/conf/wget_commands.py
testenv/misc/colour_terminal.py
testenv/server/http/http_server.py
testenv/test/base_test.py
testenv/test/http_test.py
51 files changed, 921 insertions, 156 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..2b3c5967 --- /dev/null +++ b/.gitignore @@ -0,0 +1,91 @@ +# Project Root +/.sc-start-sc_bindtextdomain +/.sc-start-sc_prohibit_HAVE_MBRTOWC +/.version +/GNUmakefile +/INSTALL +/Makefile +/Makefile.in +/aclocal.m4 +/autom4te.cache/ +/config.cache +/config.log +/config.status +/configure +/lib/ +/maint.mk +# Wildcard Ignores +*~ +po/*.gmo* +po/*.po* +src/*.o +testenv/*.log +testenv/*.trs +# build-aux/ +build-aux/.gitignore +build-aux/compile +build-aux/config.guess +build-aux/config.sub +build-aux/depcomp +build-aux/install-sh +build-aux/mdate-sh +build-aux/missing +build-aux/snippet/ +build-aux/test-driver +build-aux/texinfo.tex +# doc/ +doc/Makefile +doc/Makefile.in +doc/sample.wgetrc.munged_for_texi_inclusion +doc/stamp-vti +doc/version.texi +doc/wget.1 +doc/wget.info +doc/wget.pod +# m4/ +m4/.gitignore +m4/gnulib-cache.m4 +# po/ +po/.gitignore +po/.reference/ +po/LINGUAS +po/Makefile +po/Makefile.in +po/Makevars +po/POTFILES +po/wget.pot +# src/ +src/.deps/ +src/Makefile +src/Makefile.in +src/build_info.c +src/config.h +src/config.h.in +src/cscope.out +src/css.c +src/css_.c +src/libunittest.a +src/stamp-h1 +src/tags +src/version.c +src/wget +# testenv/ +testenv/conf/__pycache__/ +testenv/exc/__pycache__/ +testenv/misc/__pycache__/ +testenv/server/__pycache__/ +testenv/server/http/__pycache__/ +testenv/test/__pycache__/ +testenv/Makefile +testenv/Makefile.in +# tests/ +tests/Makefile +tests/Makefile.in +tests/WgetTest.pm +tests/unit-tests +# util/ +util/Makefile +util/Makefile.in +# others +/tags +/cscope.out @@ -1,3 +1,20 @@ +2014-07-25 Darshit Shah <darnir@gmail.com> + + * .gitignore: Add a gitignore file for the project. + +2013-08-31 Darshit Shah <darnir@gmail.com> + + * configure.ac: Add testenv/Makefile to AC_CONFIG_FILES. + * Makefile.am: Add testenv to SUBDIRS + +2014-07-22 Darshit Shah <darnir@gmail.com> + + * configure.ac: Fix broken code for detecting libpsl + +2014-07-21 Darshit Shah <darnir@gmail.com> + + * configure.ac: Fix check for Libpsl + 2014-06-28 Giuseppe Scrivano <gscrivan@redhat.com> * cfg.mk (local-checks-to-skip): Remove some checks. diff --git a/configure.ac b/configure.ac index 152b22f5..0f83fee7 100644 --- a/configure.ac +++ b/configure.ac @@ -61,9 +61,15 @@ dnl dnl Process features. dnl +ENABLE_PSL=no AC_ARG_WITH(libpsl, AS_HELP_STRING([--without-libpsl], - [disable support for libpsl cookie checking.])) + [disable support for libpsl cookie checking.]), + [], + [AC_SEARCH_LIBS(psl_builtin, psl, + [ENABLE_PSL=yes; AC_DEFINE([HAVE_LIBPSL], [1], [PSL Support Enabled])], + [AC_MSG_WARN(*** libpsl not found. Falling back to Wget builtin cookie checking.)]) + ]) AC_ARG_WITH(ssl, [[ --without-ssl disable SSL autodetection @@ -250,11 +256,6 @@ dnl dnl Checks for libraries. dnl -AS_IF([test x"$with_libpsl" != xno], [ - with_libpsl=yes - AC_CHECK_LIB([psl], [psl_builtin]) -]) - AS_IF([test x"$with_zlib" != xno], [ with_zlib=yes AC_CHECK_LIB(z, compress) @@ -675,7 +676,7 @@ AC_MSG_NOTICE([Summary of build options: Libs: $LIBS SSL: $with_ssl Zlib: $with_zlib - PSL: $with_libpsl + PSL: $ENABLE_PSL Digest: $ENABLE_DIGEST NTLM: $ENABLE_NTLM OPIE: $ENABLE_OPIE diff --git a/doc/ChangeLog b/doc/ChangeLog index f9967dad..d5e74961 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,3 +1,7 @@ +2014-08-03 Giuseppe Scrivano <gscrivano@gnu.org> + + * wget.texi (Download Options): Fix texinfo warning. + 2014-06-11 Giuseppe Scrivano <gscrivan@redhat.com> * Makefile.am: Remove @VAR@ with $FOO. diff --git a/doc/wget.texi b/doc/wget.texi index c487e1e1..9c2459c8 100644 --- a/doc/wget.texi +++ b/doc/wget.texi @@ -956,7 +956,8 @@ Don't use proxies, even if the appropriate @code{*_proxy} environment variable is defined. @c man end -@xref{Proxies} for more information about the use of proxies with Wget. +@xref{Proxies}, for more information about the use of proxies with +Wget. @c man begin OPTIONS @cindex quota @@ -1306,8 +1307,8 @@ really important, do not leave them lying in those files either---edit the files and delete them after Wget has started the download. @iftex -@xref{Security Considerations} for more information about security issues -with Wget. +@xref{Security Considerations}, for more information about security +issues with Wget. @end iftex @cindex Keep-Alive, turning off @@ -1843,8 +1844,8 @@ really important, do not leave them lying in those files either---edit the files and delete them after Wget has started the download. @iftex -@xref{Security Considerations} for more information about security issues -with Wget. +@xref{Security Considerations}, for more information about security +issues with Wget. @end iftex @cindex .listing files, removing diff --git a/src/ChangeLog b/src/ChangeLog index 71aef6ec..df6deabf 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -2,6 +2,23 @@ * Makefile.am (wget_SOURCES): Remove space-tab indentation. +2014-07-23 Darshit Shah <darnir@gmail.com> + + * http.c (gethttp): Fix a memory leak when retrying authorization + (gethttp): Fix memory leak when trying to parse content disposition headers + (http_loop): Assigning a new value to *local)file without freeing the old + one causes a memory leak + (http_loop): Free the HTTP message and error strings before continuing loop + +2014-07-21 Daniel Stenberg <daniel@haxx.se> + + * main.c (print_help): HTTP Method is a part of the Request not Header + +2014-07-21 Darshit Shah <darnir@gmail.com> + + * cookies.c (check_domain_match): Fix a potential memory leak when checking + cookie domain names + 2014-07-07 Tomas Hozza <thozza@redhat.com> * iri.c (locale_to_utf8): Fix checking of iconv_open return code. diff --git a/src/cookies.c b/src/cookies.c index 76301acc..bf872a88 100644 --- a/src/cookies.c +++ b/src/cookies.c @@ -546,9 +546,12 @@ check_domain_match (const char *cookie_domain, const char *host) xfree (cookie_domain_lower); xfree (host_lower); - return true ? (is_acceptable == 1) : false; + return is_acceptable == 1; no_psl: + /* Cleanup the PSL pointers first */ + xfree (cookie_domain_lower); + xfree (host_lower); #endif /* For efficiency make some elementary checks first */ @@ -2702,6 +2702,7 @@ read_header: resp_free (resp); xfree (head); xfree (auth_stat); + xfree (hs->message); goto retry_with_auth; } else @@ -2762,6 +2763,8 @@ read_header: local_file)); hs->local_file = url_file_name (u, local_file); } + + xfree_null (local_file); } /* TODO: perform this check only once. */ @@ -3726,6 +3729,8 @@ Remote file exists.\n\n")); got_name = true; *dt &= ~HEAD_ONLY; count = 0; /* the retrieve count for HEAD is reset */ + xfree_null (hstat.message); + xfree_null (hstat.error); continue; } /* send_head_first */ } /* !got_head */ @@ -3873,7 +3878,10 @@ Remote file exists.\n\n")); exit: if (ret == RETROK && local_file && !(*local_file)) - *local_file = xstrdup (hstat.local_file); + { + xfree_null (*local_file); + *local_file = xstrdup (hstat.local_file); + } free_hstat (&hstat); return ret; @@ -636,7 +636,7 @@ HTTP options:\n"), N_("\ --post-file=FILE use the POST method; send contents of FILE.\n"), N_("\ - --method=HTTPMethod use method \"HTTPMethod\" in the header.\n"), + --method=HTTPMethod use method \"HTTPMethod\" in the request.\n"), N_("\ --body-data=STRING Send STRING as data. --method MUST be set.\n"), N_("\ diff --git a/testenv/ChangeLog b/testenv/ChangeLog index 97d2266c..f9a45ab5 100644 --- a/testenv/ChangeLog +++ b/testenv/ChangeLog @@ -454,3 +454,505 @@ * Test-auth-no-challenge.py: Ensure --auth-no-challenge is honoured * Test-auth-retcode.py: Ensure correct return code after 403 Forbidden response. +2014-08-08 Darshit Shah <darnir@gmail.com> + + * conf/__init__.py: Add extra newline according to PEP8 + * conf/{authentication,expect_header,expected_files,expected_ret_code, + files_crawled,hook_sample,local_files,reject_header,response,send_header, + server_files,urls,wget_commands}.py: Add docstrings explaining the conf file + and how it should be used + * server/http/http_server (InvalidRangeHeader): Clear TODO and eliminate + this exception. Use ServerError for all such purposes. + (_Handler): Remove reference to InvalidRangeHeader + (_handler.parse_range_header): User ServerError instead of InvalidRangeHeader + (_Handler.do_GET): Add docstring + (_Handler.do_POST): Add docstring. Also create an empty dict for rules if + no rules are supplied. Send the Location header as suggested in RFC 7231 + (_Handler.do_PUT): Don't pop the server file already. Push it to later in .. + (_Handler.send_put): .. Here. If the file exists respond with a 204 No + Content message and pop the file for replacement. Do not send the + Content-Length, Content-Type headers since PUT requests should not respond + with data. + (_Handler.parse_auth_header): Fit line within 80 chars + (_Handler.check_response): Better visual indent + (_Handler.authorize_digest): Better visual indent. + (_Handler.expect_headers): Remove unused function + (_Handler.guess_type): Fix indentation + (HTTPd): Add newline according to PEP8 guidelines + (HTTPSd): Fix indentation + (StoppableHTTPServer): Add docstring + (HTTPSServer): Fix indentation + (WgetHTTPRequestHandler): Merge class into _handler. + (_Handler): Add docstring + (_Handler.parse_range_header): Fix indentation + (ServerError): Split exception into separate file ... + * exc/server_error.py: ... Here + * misc/colour_terminal.py: Add docstring, fix indentation + * test/base_test.py: Fix visual indent + * test/http_test.py: Fit within 80 char lines + +2014-08-04 Darshit Shah <darnir@gmail.com> + + * conf/server_conf.py: Delete file. Server configuration is now done via the + server_conf() method. + * server/http/http_server.py (StppableHTTPServer.server_sett): Delete + method required by the above hook + (HTTPd.server_sett): Same + +2014-07-26 Darshit Shah <darnir@gmail.com> + + * Test-*.py: Remove the '-d' switch from WGET_OPTIONS. + * test/base_test (BaseTest.gen_cmd_line): Add --debug and --no-config to the + list of switches passed to wget unconditionally. + +2014-07-23 Darshit Shah <darnir@gmail.com> + + * test/base_test.py (BaseTest.gen_cmd_line): Add support for running all + tests through valgrind if the relevant environment variable is set + * conf/expected_ret_code (ExpectedRetCode.__call__): Valgrind returns error + code 45 when it detects a memory leak. + * Readme: Update with details about valgrind tests + +2014-07-22 Darshit Shah <darnir@gmail.com> + + * (README): Remove old TODO and document SERVER_WAIT variable + +2014-06-22 Darshit Shah <darnir@gmail.com> + + * (conf.files_crawled): diff is a set object and needs explicit str + conversion. + +2014-03-13 Zihang Chen <chsc4698@gmail.com> + + * base_test.py: + (CommonMethods): Rename to BaseTest. + (BaseTest): Implement __init__ method where the class-wide variables are + initialized. Also variable names like `xxx_list` is renamed to its plural + form, e.g. `server_list` => `servers`. + (BaseTest.init_test_env): Remove name argument due to its unnecessarity. + (BaseTest.get_test_dir): Because the path of the test directory is needed + in multiple methods, this method is implemented. + (BaseTest.get_domain_addr): Rewrite the return statement utilizing str + formatting (which is more Pythonic). + (BaseTest.get_cmd_line): Rename to gen_cmd_line. Change the variables with + capitcal characters to lower ones. Also, the nested for loop is rewritten + to a plain loop using the zip function. + (BaseTest.__gen_local_filesys): Rename to gen_local_fs_snapshot. Move to + ExpectedFiles in conf/expected_files.py and is marked as a static + method. Refactor to a less verbose implementation. + (BaseTest._check_downloaded_files): Rename to __call__ to agree with the + invocation in test case classes. Move to ExpectedFiles in + conf/expected_files.py. + (BaseTest.get_server_rules): Refactor to a more Pythonic form utilizing + dict.items() and is marked static. + (BaseTest.stop_server): (new method) an abstract method which should stop + the currently using servers. + (BaseTest.instantiate_server_by): (new method) an abstract method which + should instantiate a server instance according to the given argument. + (BaseTest.__enter__): (new method) method which initialize the context + manager + (BaseTest.__exit__): (new method) method that finilize the context manager + and deal with the exceptions during the execution of the with statement, + subclasses can override this method for extensibility + * http_test.py: + (HTTPTest.__init__): Add call to super.__init__. Default values of + pre_hook, test_params, post_hook are set to None to avoid a subtle bug of + Python. Argument servers is renamed to protocols. + (HTTPTest.Server_setup): Move to BaseTest and rename to server_setup. + Calls to pre_hook_call, call_test, post_hook_call are removed. + (HTTPTest.hook_call, pre_hook_call, call_test, post_hook_call): Move to + BaseTest for that both HTTP test cases and FTP test cases may use these + methods. + (HTTPTest.init_HTTP_Server, init_HTTPS_Server): Merge and rename to + instantiate_server_by to implement the abstract method in BaseTest. + (HTTPTest.stop_HTTP_Server): Rename to stop_server to implement the + abstract method in BaseTest. Also, pull out the part where remaining + requests are gathered into a new method request_remaining. + (BaseTest.act_retcode): Rename to ret_code because ExpectedRetCode is + moved out from BaseTest, so the name act_retcode is actually a bit + verbose. + * conf/expected_ret_code.py: + (ExpectedRetCode.__call__): Rewrite the str into a more readable form. + * conf/files_crawled.py: + (FilesCrawled.__call__): Refactor this method into a more Pythonic form + utilizing the zip function. + * conf/local_files.py: + (LocalFiles__call__): Rewrite this method with the recommended with + statement. + * conf/server_conf.py: + (ServerConf.__call__): Rewrite this method due to BaseTest.server_list is + renamed to BaseTest.servers. + * conf/server_files.py: + (ServerFiles.__call__): Refactor the nested for loop into a plain one + utilizing the zip function. + * conf/urls.py: + (URLs): Rename url_list to urls. + * conf/wget_commands.py: + (WgetCommands): Rename command_list to commands, rename test_obj.options + to test_obj.wget_options. + * Test--https.py, Test-Proto.py, Test-Parallel-Proto.py: Argument servers + is changed to protocols due to change in the signature of + HTTPTest.__init__. + +2014-03-13 Zihang Chen <chsc4698@gmail.com> + + * test: (new package) package for test case classes + * WgetTest.py: Split into test/base_test.py and test/http_test.py. + * Test-*.py: Optimize the imports according to changes of WgetTest.py + +2014-03-13 Zihang Chen <chsc4698@gmail.com> + + * server: (new package) package for the server classes + * server.http: (new package) package for HTTP server + * server.ftp: (new package) package for FTP server + * HTTPServer.py: Move to server/http/http_server.py. Also change the + CERTFILE to '../certs/wget-cert.pem'. + * FTPServer.py: Move to server/ftp/ftp_server.py. + * WgetTest.py: Optimize import respect to the server classes. + +2014-03-13 Zihang Chen <chsc4698@gmail.com> + + * conf: (new package) package for rule classes and hook methods + * WgetTest.py: + (CommonMethods.Authentication): Move to conf/authentication.py. + (CommonMethods.ExpectHeader): Move to conf/expect_header.py. + (CommonMethods.RejectHeader): Move to conf/reject_header.py. + (CommonMethods.Response): Move to conf/response.py. + (CommonMethods.SendHeader): Move to conf/send_header.py. + (CommonMethods.ServerFiles): Move to conf/server_files.py. + (CommonMethods.LocalFiles): Move to conf/local_files.py. + (CommonMethods.ServerConf): Move to conf/server_conf.py. + (CommonMethods.WgetCommands): Move to conf/wget_commands.py. + (CommonMethods.Urls): Move to conf/urls.py. + (CommonMethods.ExpectedRetcode): Move to conf/expected_retcode.py. + (CommonMethods.ExpectedFiles): Move to conf/expected_files.py. + (CommonMethods.FilesCrawled): Move to conf/files_crawled.py. + (CommonMethods.__check_downloaded_files): Rename to + _check_downloaded_files, so that the method is callable from outside the + class. + (CommomMethods.get_server_rules): Modify so that it utilizes the conf + package. + (HTTPTest): Add a method hook_call(configs, name) to reduce duplications + in pre_hook_call, call_test and post_hook_call utilizing the conf package. + * conf/hook_sample.py: (new file) sample for hooks + * conf/rule_sample.py: (new file) sample for rules + * REAMDE: Update sections about customizing rules and hooks. + +2014-03-13 Zihang Chen <chsc4698@gmail.com> + + * exc: (new package) package for miscellaneous exceptions + * WgetTest.py: Move TestFailed to exc/test_failed.py. + +2014-03-13 Zihang Chen <chsc4698@gmail.com> + + * Test-Proto.py: Fix a typo (line 71: server to servers). + +2014-03-13 Zihang Chen <chsc4698@gmail.com> + + * WgetTest.py: Move WgetFile to package misc. + * README: Modify documentation respect to WgetFile. + * Test-*.py: Optimize imports about WgetFile. + +2014-03-13 Zihang Chen <chsc4698@gmail.com> + + * misc: (new package) package for miscellaneous modules + * ColourTerm.py: Move to package misc and rename to colour_terminal.py, + add print_color functions to reduce the use of string literals like + "BLUE", "RED" etc. + * WgetTest.py: + (CommonMethods.Server_setup): Change invocation to printer to print_blue. + (CommonMethods.FilesCrawled): Change invocation to printer to print_red. + (HTTPTest.__init__): Change invocations to printer to print_red and + print_green respectively. + +2014-01-02 Darshit Shah <darnir@gmail.com> + * Makefile.am: Add new Test--https.py to list of tests and EXTRA_DIST. + Also replace all tabs with spaces in file for conformity. + * Test--https.py: New test to check if Wget works correctly with HTTPS + servers + * HTTPServer.py: Import new modules for use in HTTPS Servers + (HTTPSServer): New class that generates a SSL-wrapped socket for use in a + HTTPS Server. + (HTTPSd): HTTPS daemon class. Analogous to the HTTPd class + * WgetTest.py: Define global variables HTTP and HTTPS to reflect Server + types + (CommonMethods.exec_wget): Add the protocol information to the URL before + passing it to wget + (HTTPTest.__init__): Edit syntax. The servers variable now accepts a list of + servers defined by their type. E.g. HTTP, HTTPS. + (HTTPTest.Server_setup): Reflect change in type of variable servers. + However, we maintin the value of self.servers to allow most of the code to + remain unchanged. + (HTTPTest.init_HTTPS_Server): Initialize a HTTPS Server + * Test-Parallel-Proto.py: Edit to reflect slight change in Test Fiel Syntax. + * Test-Proto.py: Same + +2014-01-02 Darshit Shah <darnir@gmail.com> + + * WgetTest.py (CommonMentods.exec_wget): Wait for n seconds before calling + the Wget executable. + +2013-12-27 Darshit Shah <darnir@gmail.com> + + * WgetTest.py: Add modeline + (CommonMethods.ServerConf): New pre-test hook that sets + BaseHTTPRequestHandler class variables in all available servers + * HTTPServer.py (HTTPd.ServerConf): Call the respective method in the Server + to set the class variables + (StoppableHTTPServer.server_sett): Set the handler class variables + +2013-12-26 Darshit Shah <darnir@gmail.com> + + * WgetTest.py (HTTPTest.call_test): Correct the call to stop_HTTP_Server. + +2013-12-25 Darshit Shah <darnir@gmail.com> + + * WgetTest.py (CommonMehtods.exec_wget): Catch and handle exception if the + Wget executable is not found at src/wget + (HTTPTest.call_test): In case of error during execution, remove all existing + servers before quitting + +2013-12-15 Darshit Shah <darnir@gmail.com> + + * WgetTest.py (HTTPTest.HTTP_setup): Rename to Server_setup so it can be + easily reused for other non-HTTP servers. + (HTTPTest.__init__): Call Server_setup instead of HTTP_setup + (HTTPTest.Server_setup): Split into three more functions, that handle + pre-hooks, test execution and post-hooks respectively. + (HTTPTest.pre_hook_call): Set up and execute the pre-test hooks. Code split + from HTTPTest.Server_setup + (HTTPTest.call_test): Execute wget and log exit code. Code split from + HTTPTest.Server_setup + (HTTPTest.post_hook_call): Set up and execute post-test hooks. Code split + from HTTPTest.Server_setup + +2013-10-14 Giuseppe Scrivano <gscrivan@redhat.com> + + * Makefile.am (XFAIL_TESTS): Remove Test--spider-r.py. + +2013-10-06 Giuseppe Scrivano <gscrivan@redhat.com> + + * Makefile.am (EXTRA_DIST): Distribute test files. + +2013-09-16 Darshit Shah <darnir@gmail.com> + + * README: Update documentation + +2013-09-14 Darshit Shah <darnir@gmail.com> + + * HTTPServer.py (StoppableHTTPServer): Define object variable + request_headers which stores a list of requests received by the server + (StoppableHTTPServer.get_req_headers): Return the list of Request + Headers stored by the server + (_Handler.do_HEAD): Send the Request MEthod string for identification + (_Handler.do_GET): Same + (_Handler.__log_request): Log the request in Request_Headers list + (_Handler.send_head): Make a call to __log_request + * Test--spider-r.py: Add new list, Request_List, which contains all + the requests that Wget is expected to send. This will allow for + fine-grained tests on recursive downloading. + * WgetTest.py (CommonMethods.FilesCrawled): New Post-Test Hook, that + ensures that all the expected Files on the server were accessed as + expected. + (HTTPTest.stop_HTTP_server): On stopping server, asks it to respond + with list of all requests it received. + +2013-09-13 Darshit Shah <darnir@gmail.com> + + * Test--spider-r.py: Test retrieval in recursive spider mode. + * Makefile.am: add new file + +2013-09-13 Darshit Shah <darnir@gmail.com> + + * HTTPServer.py (_Handler.do_HEAD): If requested path is /, respond + with /index.html + (_Handler.do_HEAD): Smartly guess value of Content-Type Header from + file extension + (_Handler.guess_type): Use a preset list of extensions and + Content-Type strings. If the extension matches one in the list, use + that string, else default to "text/plain" + +2013-09-13 Darshit Shah <darnir@gmail.com> + + * WgetTest.py (CommonMethods._replace_substring): New method that will + replace a substring delimited by {{ }} characters by the value of + self.<substring> variable + (CommonMethods.WgetCommands): Use the _replace_substring () call to + replace the substrings in the the command line. + (CommonMethods.ServerFiles): Run the _replace_substring () method on + the File contents too. + +2013-09-11 Darshit Shah <darnir@gmail.com> + + * WgetTest.py (CommonMethods.exec_wget): Expect domain_list instead of + domain. + (CommonMethods.get_cmd_line): Same. Generate command line by + prepending to each file it's respective domain string + (CommonMethods.ServerFiles): Generate file_list and server_rules for + each Server and set the config details + (HTTPTest): New named parameter, servers which signifies number of + servers to spawn + (HTTPTest.HTTP_setup): This method now takes servers as a new + parameter. Instead of storing server and domain, we now store + server_list and domain_list. Each server must be initialized through a + loop. + (HTTPTest.stop_HTTP_server): Stop all servers in a loop. + * Test-Parallel-Proto.py: Prototype test file for multiple servers. + +2013-09-10 Darshit Shah <darnir@gmail.com> + + * WgetTest.py (HTTPTest.stop_HTTP_server): With the threaded servers, + we can simply use the socketserver.shutdown() method to close the + server instead of sending a QUIT command + * HTTPServer.py (StoppabelHTTPServer.serve_forever): Delete method. No + need to override this method anymore. + (WgetHTTPRequestHandler.do_QUIT): No longer required + (HTTPd): Rename self.server to self.server_inst to reduce ambiguity + when referenced from WgetTest + +2013-09-08 Darshit Shah <darnir@gmail.com> + + * README (File Structure): Add explanation about various variables + used consistently across all tests. + +2013-09-07 Darshit Shah <darnir@gmail.com> + + * HTTPServer.py: Remove bunch of old code artefacts + * WgetTest.py: Same + +2013-09-07 Darshit Shah <darnir@gmail.com> + + * HTTPServer.py (StoppableHTTPServer.server_conf): Change global + variable fileSys to an object variable. This is good programming + practice and required for parallel-wget support. + (StoppableHTTPServer.server_forever): Edit overridden method to remove + the global queue variable. No longer required under the new working + (WgetHTTPRequestHandler.do_QUIT): Don't push fileSys through the queue + (_Handler): Rename class __Handler to _Handler to match Python's + encapsulation rules + (_Handler.do_POST): fileSys is now an object variable of the server + (_Handler.do_PUT): Same + (_Handler.send_put): Same + (_Handler.send_head): Same + (HTTPd): New class that wraps around the server for Threading + (create_server): Make new object of HTTPd. + (spawn_server): Start the thread created through create_server + (ret_fileSys): Removed method. No longer required. + * WgetTest.py (HTTPTest.__init__): Don't explicitly set + self.act_retcode. Instead toggle tests_passed boolean to set the + correct return code. + (HTTPTest.HTTP_setup): We no longer call HTTPServer.spawn_server to + start a new instance of the server. + (HTTPTest.init_HTTP_server): We no longer call the old + create_server(), spawn_server() methods. Instead use the new HTTPd + class interface to create new instances of the server + (HTTPTest.stop_HTTP_server): Don't ask server to return fileSys. + +2013-09-07 Darshit Shah <darnir@gmail.com> + + * Test-Post.py: Test basic functionality for sending HTTP POST + requests using the --method command + * Makefile.am: Add new test + +2013-09-06 Darshit Shah <darnir@gmail.com> + + * WgetTest.py (CommonMethods.__check_downloaded_files): Print a + unified diff in case there is a mismatch in the file contents + +2013-09-06 Darshit Shah <darnir@gmail.com> + + * HTTPServer.py (WgetHTTPRequestHandler.test_cookies): Comment out the + old test_cookies code. This is no longer used and was causing problems + with expected cookies. The code will soon be removed anyways + * Test-cookie.py: Add new test for basic cookie functionality + * Test-cookie-401.py: Ensure cookies are saved during a 401 response + * Test-cookie-expires.py: Ensure that the Expires field is correctly + handled + * Test-cookies-domain-mismatch.py: Ensure that mismatched domains are + handled by Wget + * Makefile.am: Add the new tests + +2013-09-06 Darshit Shah <darnir@gmail.com> + + * README: New section on pending work. Will keep updating this to keep + track of work that remains to be done on this implementation + +2013-09-05 Darshit Shah <darnir@gmail.com> + + * Test-auth-with-content-disposition.py: Add test that ensures Content + Disposition works alongwith authentication + * Makefile.am: Add new test + +2013-09-04 Darshit Shah <darnir@gmail.com> + + * Test-c-full.py: Test Continue options + * Makefile.am: Add Test-c-full.py and Test-O + +2013-09-02 Darshit Shah <darnir@gmail.com> + + * Makefile.am: Add new Test + * Test-Head.py: New Test to ensure HEAD requests are handled correctly + +2013-08-31 Darshit Shah <darnir@gmail.com> + + * README: Explain that TEST_NAME needs to be unique + * Test-auth-no-challenge.py: Edit non-unique TEST_NAME + +2013-08-31 Darshit Shah <darnir@gmail.com> + + * HTTPTest.py (ServerError): Define new Exception for handling + internal control flow. + (StoppableHTTPServer.SendHeader): Simply pass. Do nothing. Adding + functionality here seems to crash for no apparent reason. + (stoppableHTTPServer.send_cust_headers): Minor optimization. No need + for extra variable. + (__Handler.Response): Handle explicit Response Code Rules + (__Handler.Authentication): Handle Authentication rules + (__Handler.handle_auth): Actual worker method for authentication + (__Handler.ExpectHeader): Ensure Expected Headers are received + (__Handler.RejectHeader): Ensure Blacklisted Headers are NOT received + (__Handler.send_HEAD): Dynamically call server rule functions based on + the self.rules list. This feature will later be added to POST/PUT, etc + +2013-08-31 Darshit Shah <darnir@gmail.com> + + * WgetTest.py: Remove import module defaultdict. + (CommonMethods.get_server_rules): server_rules should be a dict, not a + defaultdict (list). + * HTTPServer.py (WgetHTTPRequestHandler.get_rule_list): If rule does + not exist, return None. Not an emppty list. + (WgetHTTPRequestHandler.test_cookies): Rule variable is not a list + (__Handler.send_cust_headers): Same + (__Handler.custom_response): Same + (__Handler.is_authorized): Same + (__Handler.expect_headers): Same + (__Handler.reject_headers): Same + +2013-08-31 Darshit Shah <darnir@gmail.com> + + * README: (newfile) Simple help / instructions about using the Test + Environment. + * Makefile.am: (newfile) Makefile for the Test Environment. Uses the + Automake Parallel Test Harness + * WgetTest.py: (newfile) Base module that executes the Test. + * HTTPServer.py: (newfile) Contains the custom HTTP Server for the + Test Environment. Creates an instance of http.server in Python3. + * FTPServer.py: (newfile) Overrides methods from pyftpdlib for use in + the Test Environment. ** Work under progress **. + * ColourTerm.py: (newfile) A custom module to output coloured text to + the terminal. Known to work on POSIX shells. + * Test-Proto.py: (newfile) A prototype Test File. This should be + copied when writing a new Test Case. + * Test-Content-disposition-2.py: Test Content Disposition clobbering + * Test-Content-disposition.py: Test Content Disposition Headers + * Test-O.py: Test Output filename command + * Test-auth-basic-fail.py: Test returncode on auth failure + * Test-auth-basic.py: Test Basic Auth negotiation + * Test-auth-both.py: Test handling of Multiple auth providers. This + test currently fails. + * Test-auth-digest.py: Test Digest Auth Negotiation + * Test-auth-no-challenge-url.py: Ensure --auth-no-challenge is handled + when auth details are in-URL. + * Test-auth-no-challenge.py: Ensure --auth-no-challenge is honoured + * Test-auth-retcode.py: Ensure correct return code after 403 Forbidden + response. diff --git a/testenv/Test--https.py b/testenv/Test--https.py index 55f417be..17f49d3d 100755 --- a/testenv/Test--https.py +++ b/testenv/Test--https.py @@ -17,7 +17,7 @@ A_File = WgetFile ("File1", File1) B_File = WgetFile ("File2", File2) C_File = WgetFile ("File3", File3) -WGET_OPTIONS = "-d --no-check-certificate" +WGET_OPTIONS = "--no-check-certificate" WGET_URLS = [["File1", "File2"]] Files = [[A_File, B_File]] diff --git a/testenv/Test--spider-r.py b/testenv/Test--spider-r.py index df023d3a..5eb01e46 100755 --- a/testenv/Test--spider-r.py +++ b/testenv/Test--spider-r.py @@ -74,7 +74,7 @@ Request_List = [ ] ] -WGET_OPTIONS = "-d --spider -r" +WGET_OPTIONS = "--spider -r" WGET_URLS = [[""]] Files = [[index_html, secondpage_html, thirdpage_html, dummy_txt]] diff --git a/testenv/Test-Content-disposition-2.py b/testenv/Test-Content-disposition-2.py index 3bf49405..5b9cf91a 100755 --- a/testenv/Test-Content-disposition-2.py +++ b/testenv/Test-Content-disposition-2.py @@ -20,7 +20,7 @@ File2_rules = { A_File = WgetFile ("HTTP.Teapot", File1) B_File = WgetFile ("File2", File2, rules=File2_rules) -WGET_OPTIONS = "-d --content-disposition" +WGET_OPTIONS = "--content-disposition" WGET_URLS = [["File2"]] Files = [[B_File]] diff --git a/testenv/Test-Content-disposition.py b/testenv/Test-Content-disposition.py index ce245994..027a9e8c 100755 --- a/testenv/Test-Content-disposition.py +++ b/testenv/Test-Content-disposition.py @@ -25,7 +25,7 @@ File1_rules = { } A_File = WgetFile ("LOTR", File1, rules=File1_rules) -WGET_OPTIONS = "-d --content-disposition" +WGET_OPTIONS = "--content-disposition" WGET_URLS = [["LOTR"]] Files = [[A_File]] diff --git a/testenv/Test-Head.py b/testenv/Test-Head.py index e3562529..77e5c575 100755 --- a/testenv/Test-Head.py +++ b/testenv/Test-Head.py @@ -13,7 +13,7 @@ File1 = "You shall not pass!" A_File = WgetFile ("File1", File1) -WGET_OPTIONS = "-d --method=HEAD" +WGET_OPTIONS = "--method=HEAD" WGET_URLS = [["File1"]] Files = [[A_File]] diff --git a/testenv/Test-O.py b/testenv/Test-O.py index 784a229d..90cb146a 100755 --- a/testenv/Test-O.py +++ b/testenv/Test-O.py @@ -13,7 +13,7 @@ File1 = "Test Contents." A_File = WgetFile ("File1", File1) -WGET_OPTIONS = "-d -O NewFile.txt" +WGET_OPTIONS = "-O NewFile.txt" WGET_URLS = [["File1"]] Files = [[A_File]] diff --git a/testenv/Test-Parallel-Proto.py b/testenv/Test-Parallel-Proto.py index b5e42bbe..beb768c5 100755 --- a/testenv/Test-Parallel-Proto.py +++ b/testenv/Test-Parallel-Proto.py @@ -18,7 +18,7 @@ A_File = WgetFile ("File1", File1) B_File = WgetFile ("File2", File2) C_File = WgetFile ("File3", File3) -WGET_OPTIONS = "-d" +WGET_OPTIONS = "" WGET_URLS = [["File1"], ["File2"]] Files = [[A_File], [B_File]] diff --git a/testenv/Test-Post.py b/testenv/Test-Post.py index 8983454b..223641af 100755 --- a/testenv/Test-Post.py +++ b/testenv/Test-Post.py @@ -17,7 +17,7 @@ TestMessage""" A_File = WgetFile ("File1", File1) -WGET_OPTIONS = "-d --method=post --body-data=TestMessage" +WGET_OPTIONS = "--method=post --body-data=TestMessage" WGET_URLS = [["File1"]] Files = [[A_File]] diff --git a/testenv/Test-Proto.py b/testenv/Test-Proto.py index d26b2bb3..cc71fb9d 100755 --- a/testenv/Test-Proto.py +++ b/testenv/Test-Proto.py @@ -39,7 +39,7 @@ A_File = WgetFile ("File1", File1, rules=File1_rules) B_File = WgetFile ("File2", File2, rules=File2_rules) C_File = WgetFile ("File3", File3) -WGET_OPTIONS = "-d --content-disposition --user=Sauron --password=TheEye" +WGET_OPTIONS = "--content-disposition --user=Sauron --password=TheEye" WGET_URLS = [["File1", "File2"]] Servers = [HTTP] diff --git a/testenv/Test-auth-basic-fail.py b/testenv/Test-auth-basic-fail.py index 263f7566..0badf693 100755 --- a/testenv/Test-auth-basic-fail.py +++ b/testenv/Test-auth-basic-fail.py @@ -20,7 +20,7 @@ File1_rules = { } A_File = WgetFile ("File1", File1, rules=File1_rules) -WGET_OPTIONS = "-d --user=Sauron --password=Eye" +WGET_OPTIONS = "--user=Sauron --password=Eye" WGET_URLS = [["File1"]] Files = [[A_File]] diff --git a/testenv/Test-auth-basic.py b/testenv/Test-auth-basic.py index 102bf8c6..49bb27b2 100755 --- a/testenv/Test-auth-basic.py +++ b/testenv/Test-auth-basic.py @@ -28,7 +28,7 @@ File2_rules = { A_File = WgetFile ("File1", File1, rules=File1_rules) B_File = WgetFile ("File2", File2, rules=File2_rules) -WGET_OPTIONS = "-d --user=Sauron --password=TheEye" +WGET_OPTIONS = "--user=Sauron --password=TheEye" WGET_URLS = [["File1", "File2"]] Files = [[A_File, B_File]] diff --git a/testenv/Test-auth-both.py b/testenv/Test-auth-both.py index 2da2840f..91d72b84 100755 --- a/testenv/Test-auth-both.py +++ b/testenv/Test-auth-both.py @@ -46,7 +46,7 @@ A_File = WgetFile ("File1", File1, rules=File1_rules) B_File = WgetFile ("File2", File2, rules=File2_rules) C_File = WgetFile ("File3", File3, rules=File3_rules) -WGET_OPTIONS = "-d --user=Sauron --password=TheEye" +WGET_OPTIONS = "--user=Sauron --password=TheEye" WGET_URLS = [["File1", "File2", "File3"]] Files = [[A_File, B_File, C_File]] diff --git a/testenv/Test-auth-digest.py b/testenv/Test-auth-digest.py index f6d28c7c..8a73c0d5 100755 --- a/testenv/Test-auth-digest.py +++ b/testenv/Test-auth-digest.py @@ -19,7 +19,7 @@ File1_rules = { } A_File = WgetFile ("File1", File1, rules=File1_rules) -WGET_OPTIONS = "-d --user=Pacman --password=Omnomnom" +WGET_OPTIONS = "--user=Pacman --password=Omnomnom" WGET_URLS = [["File1"]] Files = [[A_File]] diff --git a/testenv/Test-auth-no-challenge-url.py b/testenv/Test-auth-no-challenge-url.py index c39ebaa9..9e062600 100755 --- a/testenv/Test-auth-no-challenge-url.py +++ b/testenv/Test-auth-no-challenge-url.py @@ -23,7 +23,7 @@ File1_rules = { } A_File = WgetFile ("File1", File1, rules=File1_rules) -WGET_OPTIONS = "-d --auth-no-challenge http://Pacman:Omnomnom@localhost:{{port}}/File1" +WGET_OPTIONS = "--auth-no-challenge http://Pacman:Omnomnom@localhost:{{port}}/File1" WGET_URLS = [[]] Files = [[A_File]] diff --git a/testenv/Test-auth-no-challenge.py b/testenv/Test-auth-no-challenge.py index f02c0307..d155d725 100755 --- a/testenv/Test-auth-no-challenge.py +++ b/testenv/Test-auth-no-challenge.py @@ -23,7 +23,7 @@ File1_rules = { } A_File = WgetFile ("File1", File1, rules=File1_rules) -WGET_OPTIONS = "-d --auth-no-challenge --user=Pacman --password=Omnomnom" +WGET_OPTIONS = "--auth-no-challenge --user=Pacman --password=Omnomnom" WGET_URLS = [["File1"]] Files = [[A_File]] diff --git a/testenv/Test-auth-retcode.py b/testenv/Test-auth-retcode.py index 8719bd01..13eb583b 100755 --- a/testenv/Test-auth-retcode.py +++ b/testenv/Test-auth-retcode.py @@ -19,7 +19,7 @@ File1_rules = { A_File = WgetFile ("File1", File1, rules=File1_rules) -WGET_OPTIONS = "-d" +WGET_OPTIONS = "" WGET_URLS = [["File1"]] Files = [[A_File]] diff --git a/testenv/Test-auth-with-content-disposition.py b/testenv/Test-auth-with-content-disposition.py index f74a9592..df28c91e 100755 --- a/testenv/Test-auth-with-content-disposition.py +++ b/testenv/Test-auth-with-content-disposition.py @@ -23,7 +23,7 @@ File1_rules = { } A_File = WgetFile ("File1", File1, rules=File1_rules) -WGET_OPTIONS = "-d --user=Pacman --password=Omnomnom --content-disposition" +WGET_OPTIONS = "--user=Pacman --password=Omnomnom --content-disposition" WGET_URLS = [["File1"]] Files = [[A_File]] diff --git a/testenv/Test-c-full.py b/testenv/Test-c-full.py index 3cdcd768..6dc3f5e2 100755 --- a/testenv/Test-c-full.py +++ b/testenv/Test-c-full.py @@ -20,7 +20,7 @@ D_File = WgetFile ("File2", File2) E_File = WgetFile ("File3", File1) -WGET_OPTIONS = "-d -c" +WGET_OPTIONS = "-c" WGET_URLS = [["File1", "File2", "File3"]] Files = [[A_File, C_File, E_File]] diff --git a/testenv/Test-cookie-401.py b/testenv/Test-cookie-401.py index 9488c34e..4ffb1ff9 100755 --- a/testenv/Test-cookie-401.py +++ b/testenv/Test-cookie-401.py @@ -28,7 +28,7 @@ File2_rules = { A_File = WgetFile ("File1", File1, rules=File1_rules) B_File = WgetFile ("File2", File2, rules=File2_rules) -WGET_OPTIONS = "-d" +WGET_OPTIONS = "" WGET_URLS = [["File1", "File2"]] Files = [[A_File, B_File]] diff --git a/testenv/Test-cookie-domain-mismatch.py b/testenv/Test-cookie-domain-mismatch.py index 92487f43..45c5f1de 100755 --- a/testenv/Test-cookie-domain-mismatch.py +++ b/testenv/Test-cookie-domain-mismatch.py @@ -26,7 +26,7 @@ File2_rules = { A_File = WgetFile ("File1", File1, rules=File1_rules) B_File = WgetFile ("File2", File2, rules=File2_rules) -WGET_OPTIONS = "-d" +WGET_OPTIONS = "" WGET_URLS = [["File1", "File2"]] Files = [[A_File, B_File]] diff --git a/testenv/Test-cookie-expires.py b/testenv/Test-cookie-expires.py index 48a93b99..7ad919d5 100755 --- a/testenv/Test-cookie-expires.py +++ b/testenv/Test-cookie-expires.py @@ -49,7 +49,7 @@ B_File = WgetFile ("File2", File2, rules=File2_rules) C_File = WgetFile ("File3", File3, rules=File3_rules) D_File = WgetFile ("File4", File4, rules=File4_rules) -WGET_OPTIONS = "-d" +WGET_OPTIONS = "" WGET_URLS = [["File1", "File2", "File3", "File4"]] Files = [[A_File, B_File, C_File, D_File]] diff --git a/testenv/Test-cookie.py b/testenv/Test-cookie.py index b70316d8..13935181 100755 --- a/testenv/Test-cookie.py +++ b/testenv/Test-cookie.py @@ -26,7 +26,7 @@ File2_rules = { A_File = WgetFile ("File1", File1, rules=File1_rules) B_File = WgetFile ("File2", File2, rules=File2_rules) -WGET_OPTIONS = "-d" +WGET_OPTIONS = "" WGET_URLS = [["File1", "File2"]] Files = [[A_File, B_File]] diff --git a/testenv/conf/__init__.py b/testenv/conf/__init__.py index 156e9b60..603bd62b 100644 --- a/testenv/conf/__init__.py +++ b/testenv/conf/__init__.py @@ -3,6 +3,7 @@ import os # this file implements the mechanism of conf class auto-registration, # don't modify this file if you have no idea what you're doing + def gen_hook(): hook_table = {} diff --git a/testenv/conf/authentication.py b/testenv/conf/authentication.py index 58cbaff0..c87994ad 100644 --- a/testenv/conf/authentication.py +++ b/testenv/conf/authentication.py @@ -1,5 +1,18 @@ from conf import rule +""" Rule: Authentication +This file defines an authentication rule which when applied to any file will +cause the server to prompt the client for the required authentication details +before serving it. +auth_type must be either of: Basic, Digest, Both or Both-inline +When auth_type is Basic or Digest, the server asks for the respective +authentication in its response. When auth_type is Both, the server sends two +Authenticate headers, one requesting Basic and the other requesting Digest +authentication. If auth_type is Both-inline, the server sends only one +Authenticate header, but lists both Basic and Digest as supported mechanisms in +that. +""" + @rule() class Authentication: diff --git a/testenv/conf/expect_header.py b/testenv/conf/expect_header.py index 87b0e24d..055099fa 100644 --- a/testenv/conf/expect_header.py +++ b/testenv/conf/expect_header.py @@ -1,5 +1,10 @@ from conf import rule +""" Rule: ExpectHeader +This rule defines a dictionary of headers and their value which the server +should expect in each request for the file to which the rule was applied. +""" + @rule() class ExpectHeader: diff --git a/testenv/conf/expected_files.py b/testenv/conf/expected_files.py index a8b2ee19..2c8d632c 100644 --- a/testenv/conf/expected_files.py +++ b/testenv/conf/expected_files.py @@ -4,6 +4,15 @@ import sys from conf import hook from exc.test_failed import TestFailed +""" Post-Test Hook: ExpectedFiles +This is a Post-Test hook that checks the test directory for the files it +contains. A dictionary object is passed to it, which contains a mapping of +filenames and contents of all the files that the directory is expected to +contain. +Raises a TestFailed exception if the expected files are not found or if extra +files are found, else returns gracefully. +""" + @hook() class ExpectedFiles: @@ -34,7 +43,7 @@ class ExpectedFiles: fromfile='Actual', tofile='Expected'): print(line, file=sys.stderr) - raise TestFailed('Contents of %s do not match.' % file.name) + raise TestFailed('Contents of %s do not match' % file.name) else: raise TestFailed('Expected file %s not found.' % file.name) if local_fs: diff --git a/testenv/conf/expected_ret_code.py b/testenv/conf/expected_ret_code.py index febef32f..87cba13b 100644 --- a/testenv/conf/expected_ret_code.py +++ b/testenv/conf/expected_ret_code.py @@ -1,6 +1,14 @@ from exc.test_failed import TestFailed from conf import hook +""" Post-Test Hook: ExpectedRetCode +This is a post-test hook which checks if the exit code of the Wget instance +under test is the same as that expected. As a result, this is a very important +post test hook which is checked in all the tests. +Returns a TestFailed exception if the return code does not match the expected +value. Else returns gracefully. +""" + @hook(alias='ExpectedRetcode') class ExpectedRetCode: diff --git a/testenv/conf/files_crawled.py b/testenv/conf/files_crawled.py index 3f52008d..334e5964 100644 --- a/testenv/conf/files_crawled.py +++ b/testenv/conf/files_crawled.py @@ -2,6 +2,15 @@ from misc.colour_terminal import print_red from conf import hook from exc.test_failed import TestFailed +""" Post-Test Hook: FilesCrawled +This is a post test hook that is invoked in tests that check wget's behaviour +in recursive mode. It expects an ordered list of the request lines that Wget +must send to the server. If the requests received by the server do not match +the provided list, IN THE GIVEN ORDER, then it raises a TestFailed exception. +Such a test can be used to check the implementation of the recursion algorithm +in Wget too. +""" + @hook() class FilesCrawled: diff --git a/testenv/conf/hook_sample.py b/testenv/conf/hook_sample.py index f48942f0..6230a706 100644 --- a/testenv/conf/hook_sample.py +++ b/testenv/conf/hook_sample.py @@ -1,7 +1,12 @@ from exc.test_failed import TestFailed from conf import hook -# this file is a hook example +""" Hook: SampleHook +This a sample file for how a new hook should be defined. +Any errors should always be reported by raising a TestFailed exception instead +of returning a true or false value. +""" + @hook(alias='SampleHookAlias') class SampleHook: @@ -12,4 +17,6 @@ class SampleHook: def __call__(self, test_obj): # implement hook here # if you need the test case instance, refer to test_obj + if False: + raise TestFailed ("Reason") pass diff --git a/testenv/conf/local_files.py b/testenv/conf/local_files.py index 1eb3e4e5..5f9c8fad 100644 --- a/testenv/conf/local_files.py +++ b/testenv/conf/local_files.py @@ -1,5 +1,11 @@ from conf import hook +""" Pre-Test Hook: LocalFiles +This is a pre-test hook used to generate the specific environment before a test +is run. The LocalFiles hook creates the files which should exist on disk before +invoking Wget. +""" + @hook() class LocalFiles: diff --git a/testenv/conf/reject_header.py b/testenv/conf/reject_header.py index 1f451456..53e237dd 100644 --- a/testenv/conf/reject_header.py +++ b/testenv/conf/reject_header.py @@ -1,5 +1,11 @@ from conf import rule +""" Rule: RejectHeader +This is a server side rule which expects a dictionary object of Headers and +their values which should be blacklisted by the server for a particular file's +requests. +""" + @rule() class RejectHeader: diff --git a/testenv/conf/response.py b/testenv/conf/response.py index 23d55de5..976a9ce9 100644 --- a/testenv/conf/response.py +++ b/testenv/conf/response.py @@ -1,5 +1,9 @@ from conf import rule +""" Rule: Response +When this rule is set against a certain file, the server will unconditionally +respond to any request for the said file with the provided response code. """ + @rule() class Response: diff --git a/testenv/conf/send_header.py b/testenv/conf/send_header.py index 61dbc0ec..1ac54cc4 100644 --- a/testenv/conf/send_header.py +++ b/testenv/conf/send_header.py @@ -1,5 +1,10 @@ from conf import rule +""" Rule: SendHeader +Have the server send custom headers when responding to a request for the file +this rule is applied to. The header_obj object is expected to be dictionary +mapping headers to their contents. """ + @rule() class SendHeader: diff --git a/testenv/conf/server_files.py b/testenv/conf/server_files.py index bf6c1633..1e9d3466 100644 --- a/testenv/conf/server_files.py +++ b/testenv/conf/server_files.py @@ -1,5 +1,16 @@ from conf import hook +""" Pre-Test Hook: ServerFiles +This hook is used to define a set of files on the server's virtual filesystem. +server_files is expected to be dictionary that maps filenames to their +contents. In the future, this can be used to add additional metadat to the +files using the WgetFile class too. + +This hook also does some additional processing on the contents of the file. Any +text between {{and}} is replaced by the contents of a class variable of the +same name. This is useful in creating files that contain an absolute link to +another file on the same server. """ + @hook() class ServerFiles: diff --git a/testenv/conf/urls.py b/testenv/conf/urls.py index 60015867..f34c13eb 100644 --- a/testenv/conf/urls.py +++ b/testenv/conf/urls.py @@ -1,5 +1,9 @@ from conf import hook +""" Pre-Test Hook: URLS +This hook is used to define the paths of the files on the test server that wget +will send a request for. """ + @hook(alias='Urls') class URLs: diff --git a/testenv/conf/wget_commands.py b/testenv/conf/wget_commands.py index a326bb56..2b7522ec 100644 --- a/testenv/conf/wget_commands.py +++ b/testenv/conf/wget_commands.py @@ -1,5 +1,10 @@ from conf import hook +""" Pre-Test Hook: WgetCommands +This hook is used to specify the test specific switches that must be passed to +wget on invokation. Default switches are hard coded in the test suite itself. +""" + @hook() class WgetCommands: diff --git a/testenv/exc/server_error.py b/testenv/exc/server_error.py new file mode 100644 index 00000000..b8a37ce7 --- /dev/null +++ b/testenv/exc/server_error.py @@ -0,0 +1,7 @@ + +class ServerError (Exception): + """ A custom exception which is raised by the test servers. Often used to + handle control flow. """ + + def __init__ (self, err_message): + self.err_message = err_message diff --git a/testenv/misc/colour_terminal.py b/testenv/misc/colour_terminal.py index 206ffa30..cfbae94e 100644 --- a/testenv/misc/colour_terminal.py +++ b/testenv/misc/colour_terminal.py @@ -2,24 +2,39 @@ from functools import partial import platform from os import getenv +""" This module allows printing coloured output to the terminal when running a +Wget Test under certain conditions. +The output is coloured only on Linux systems. This is because coloured output +in the terminal on Windows requires too much effort for what is simply a +convenience. This might work on OSX terminals, but without a confirmation, it +remains unsupported. + +Another important aspect is that the coloured output is printed only if the +environment variable MAKE_CHECK is not set. This variable is set when running +the test suite through, `make check`. In that case, the output is not only +printed to the terminal but also copied to a log file where the ANSI escape +codes on;y add clutter. """ + + T_COLORS = { - 'PURPLE' : '\033[95m', - 'BLUE' : '\033[94m', - 'GREEN' : '\033[92m', - 'YELLOW' : '\033[93m', - 'RED' : '\033[91m', - 'ENDC' : '\033[0m' + 'PURPLE' : '\033[95m', + 'BLUE' : '\033[94m', + 'GREEN' : '\033[92m', + 'YELLOW' : '\033[93m', + 'RED' : '\033[91m', + 'ENDC' : '\033[0m' } + def printer (color, string): - if platform.system () == 'Linux': - if getenv ("MAKE_CHECK", "False") == "True": - print (string) - else: - print (T_COLORS.get (color) + string + T_COLORS.get ('ENDC')) + if platform.system () == 'Linux': + if getenv ("MAKE_CHECK", "False") == "True": + print (string) + else: + print (T_COLORS.get (color) + string + T_COLORS.get ('ENDC')) - else: - print (string) + else: + print (string) print_blue = partial(printer, 'BLUE') @@ -28,4 +43,4 @@ print_green = partial(printer, 'GREEN') print_purple = partial(printer, 'PURPLE') print_yellow = partial(printer, 'YELLOW') -# vim: set ts=8 sw=3 tw=0 et : +# vim: set ts=8 sw=3 tw=80 et : diff --git a/testenv/server/http/http_server.py b/testenv/server/http/http_server.py index e554a105..12e04348 100644 --- a/testenv/server/http/http_server.py +++ b/testenv/server/http/http_server.py @@ -1,4 +1,5 @@ from http.server import HTTPServer, BaseHTTPRequestHandler +from exc.server_error import ServerError from socketserver import BaseServer from posixpath import basename, splitext from base64 import b64encode @@ -11,20 +12,12 @@ import ssl import os -class InvalidRangeHeader (Exception): - - """ Create an Exception for handling of invalid Range Headers. """ - # TODO: Eliminate this exception and use only ServerError - - def __init__ (self, err_message): - self.err_message = err_message - -class ServerError (Exception): - def __init__ (self, err_message): - self.err_message = err_message - - class StoppableHTTPServer (HTTPServer): + """ This class extends the HTTPServer class from default http.server library + in Python 3. The StoppableHTTPServer class is capable of starting an HTTP + server that serves a virtual set of files made by the WgetFile class and + has most of its properties configurable through the server_conf() + method. """ request_headers = list () @@ -35,45 +28,45 @@ class StoppableHTTPServer (HTTPServer): self.server_configs = conf_dict self.fileSys = filelist - def server_sett (self, settings): - for settings_key in settings: - setattr (self.RequestHandlerClass, settings_key, settings[settings_key]) - def get_req_headers (self): return self.request_headers -class HTTPSServer (StoppableHTTPServer): - def __init__ (self, address, handler): - BaseServer.__init__ (self, address, handler) - print (os.getcwd()) - CERTFILE = os.path.abspath (os.path.join ('..', 'certs', 'wget-cert.pem')) - print (CERTFILE) - fop = open (CERTFILE) - print (fop.readline()) - self.socket = ssl.wrap_socket ( - sock = socket.socket (self.address_family, self.socket_type), - ssl_version = ssl.PROTOCOL_TLSv1, - certfile = CERTFILE, - server_side = True - ) - self.server_bind () - self.server_activate () - -class WgetHTTPRequestHandler (BaseHTTPRequestHandler): - - """ Define methods for handling Test Checks. """ +class HTTPSServer (StoppableHTTPServer): + """ The HTTPSServer class extends the StoppableHTTPServer class with + additional support for secure connections through SSL. """ + + def __init__ (self, address, handler): + BaseServer.__init__ (self, address, handler) + print (os.getcwd()) + CERTFILE = os.path.abspath(os.path.join('..', 'certs', 'wget-cert.pem')) + print (CERTFILE) + fop = open (CERTFILE) + print (fop.readline()) + self.socket = ssl.wrap_socket ( + sock = socket.socket (self.address_family, self.socket_type), + ssl_version = ssl.PROTOCOL_TLSv1, + certfile = CERTFILE, + server_side = True + ) + self.server_bind() + self.server_activate() + + +class _Handler (BaseHTTPRequestHandler): + """ This is a private class which tells the server *HOW* to handle each + request. For each HTTP Request Command that the server should be capable of + responding to, there must exist a do_REQUESTNAME() method which details the + steps in which such requests should be processed. The rest of the methods + in this class are auxilliary methods created to help in processing certain + requests. """ def get_rule_list (self, name): r_list = self.rules.get (name) if name in self.rules else None return r_list - -class _Handler (WgetHTTPRequestHandler): - - """ Define Handler Methods for different Requests. """ - - InvalidRangeHeader = InvalidRangeHeader + # The defailt protocol version of the server we run is HTTP/1.1 not + # HTTP/1.0 which is the default with the http.server module. protocol_version = 'HTTP/1.1' """ Define functions for various HTTP Requests. """ @@ -82,6 +75,11 @@ class _Handler (WgetHTTPRequestHandler): self.send_head ("HEAD") def do_GET (self): + """ Process HTTP GET requests. This is the same as processing HEAD + requests and then actually transmitting the data to the client. If + send_head() does not specify any "start" offset, we send the complete + data, else transmit only partial data. """ + content, start = self.send_head ("GET") if content: if start is None: @@ -90,11 +88,26 @@ class _Handler (WgetHTTPRequestHandler): self.wfile.write (content.encode ('utf-8')[start:]) def do_POST (self): + """ According to RFC 7231 sec 4.3.3, if the resource requested in a POST + request does not exist on the server, the first POST request should + create that resource. PUT requests are otherwise used to create a + resource. Hence, we call the handle for processing PUT requests if the + resource requested does not already exist. + + Currently, when the server recieves a POST request for a resource, we + simply append the body data to the existing file and return the new + file to the client. If the file does not exist, a new file is created + using the contents of the request body. """ + path = self.path[1:] - self.rules = self.server.server_configs.get (path) - if not self.custom_response (): - return (None, None) if path in self.server.fileSys: + self.rules = self.server.server_configs.get (path) + if not self.rules: + self.rules = dict () + + if not self.custom_response (): + return (None, None) + body_data = self.get_body_data () self.send_response (200) self.send_header ("Content-type", "text/plain") @@ -102,6 +115,7 @@ class _Handler (WgetHTTPRequestHandler): total_length = len (content) self.server.fileSys[path] = content self.send_header ("Content-Length", total_length) + self.send_header ("Location", self.path) self.finish_headers () try: self.wfile.write (content.encode ('utf-8')) @@ -115,7 +129,6 @@ class _Handler (WgetHTTPRequestHandler): self.rules = self.server.server_configs.get (path) if not self.custom_response (): return (None, None) - self.server.fileSys.pop (path, None) self.send_put (path) """ End of HTTP Request Method Handlers. """ @@ -126,12 +139,12 @@ class _Handler (WgetHTTPRequestHandler): if header_line is None: return None if not header_line.startswith ("bytes="): - raise InvalidRangeHeader ("Cannot parse header Range: %s" % - (header_line)) + raise ServerError ("Cannot parse header Range: %s" % + (header_line)) regex = re.match (r"^bytes=(\d*)\-$", header_line) range_start = int (regex.group (1)) if range_start >= length: - raise InvalidRangeHeader ("Range Overflow") + raise ServerError ("Range Overflow") return range_start def get_body_data (self): @@ -141,23 +154,27 @@ class _Handler (WgetHTTPRequestHandler): return body_data def send_put (self, path): + if path in self.server.fileSys: + self.server.fileSys.pop (path, None) + self.send_response (204) + else: + self.rules = dict () + self.send_response (201) body_data = self.get_body_data () - self.send_response (201) self.server.fileSys[path] = body_data - self.send_header ("Content-type", "text/plain") - self.send_header ("Content-Length", len (body_data)) + self.send_header ("Location", self.path) self.finish_headers () - try: - self.wfile.write (body_data.encode ('utf-8')) - except Exception: - pass + """ This empty method is called automatically when all the rules are + processed for a given request. However, send_header() should only be called + AFTER a response has been sent. But, at the moment of processing the rules, + the appropriate response has not yet been identified. As a result, we defer + the processing of this rule till later. Each do_* request handler MUST call + finish_headers() instead of end_headers(). The finish_headers() method + takes care of sending the appropriate headers before completing the + response. """ def SendHeader (self, header_obj): pass -# headers_list = header_obj.headers -# for header_line in headers_list: -# print (header_line + " : " + headers_list[header_line]) -# self.send_header (header_line, headers_list[header_line]) def send_cust_headers (self): header_obj = self.get_rule_list ('SendHeader') @@ -195,11 +212,11 @@ class _Handler (WgetHTTPRequestHandler): if auth_type == "Basic": challenge_str = 'Basic realm="Wget-Test"' elif auth_type == "Digest" or auth_type == "Both_inline": - self.nonce = md5 (str (random ()).encode ('utf-8')).hexdigest () - self.opaque = md5 (str (random ()).encode ('utf-8')).hexdigest () - challenge_str = 'Digest realm="Test", nonce="%s", opaque="%s"' %( - self.nonce, - self.opaque) + self.nonce = md5 (str (random ()).encode ('utf-8')).hexdigest() + self.opaque = md5 (str (random ()).encode ('utf-8')).hexdigest() + challenge_str = 'Digest realm="Test", nonce="%s", opaque="%s"' % ( + self.nonce, + self.opaque) challenge_str += ', qop="auth"' if auth_type == "Both_inline": challenge_str = 'Basic realm="Wget-Test", ' + challenge_str @@ -218,9 +235,9 @@ class _Handler (WgetHTTPRequestHandler): n = len("Digest ") auth_header = auth_header[n:].strip() items = auth_header.split(", ") - key_values = [i.split("=", 1) for i in items] - key_values = [(k.strip(), v.strip().replace('"', '')) for k, v in key_values] - return dict(key_values) + keyvals = [i.split("=", 1) for i in items] + keyvals = [(k.strip(), v.strip().replace('"', '')) for k, v in keyvals] + return dict(keyvals) def KD (self, secret, data): return self.H (secret + ":" + data) @@ -237,10 +254,10 @@ class _Handler (WgetHTTPRequestHandler): def check_response (self, params): if "qop" in params: data_str = params['nonce'] \ - + ":" + params['nc'] \ - + ":" + params['cnonce'] \ - + ":" + params['qop'] \ - + ":" + self.H (self.A2 (params)) + + ":" + params['nc'] \ + + ":" + params['cnonce'] \ + + ":" + params['qop'] \ + + ":" + self.H (self.A2 (params)) else: data_str = params['nonce'] + ":" + self.H (self.A2 (params)) resp = self.KD (self.H (self.A1 ()), data_str) @@ -256,11 +273,12 @@ class _Handler (WgetHTTPRequestHandler): params = self.parse_auth_header (auth_header) pass_auth = True if self.user != params['username'] or \ - self.nonce != params['nonce'] or self.opaque != params['opaque']: + self.nonce != params['nonce'] or \ + self.opaque != params['opaque']: pass_auth = False req_attribs = ['username', 'realm', 'nonce', 'uri', 'response'] for attrib in req_attribs: - if not attrib in params: + if attrib not in params: pass_auth = False if not self.check_response (params): pass_auth = False @@ -326,19 +344,6 @@ class _Handler (WgetHTTPRequestHandler): self.finish_headers () raise ServerError ("Header " + header_line + " not found") - def expect_headers (self): - """ This is modified code to handle a few changes. Should be removed ASAP """ - exp_headers_obj = self.get_rule_list ('ExpectHeader') - if exp_headers_obj: - exp_headers = exp_headers_obj.headers - for header_line in exp_headers: - header_re = self.headers.get (header_line) - if header_re is None or header_re != exp_headers[header_line]: - self.send_error (400, 'Expected Header not Found') - self.end_headers () - return False - return True - def RejectHeader (self, header_obj): rej_headers = header_obj.headers for header_line in rej_headers: @@ -400,7 +405,7 @@ class _Handler (WgetHTTPRequestHandler): try: self.range_begin = self.parse_range_header ( self.headers.get ("Range"), content_length) - except InvalidRangeHeader as ae: + except ServerError as ae: # self.log_error("%s", ae.err_message) if ae.err_message == "Range Overflow": self.send_response (416) @@ -431,9 +436,9 @@ class _Handler (WgetHTTPRequestHandler): base_name = basename ("/" + path) name, ext = splitext (base_name) extension_map = { - ".txt" : "text/plain", - ".css" : "text/css", - ".html" : "text/html" + ".txt" : "text/plain", + ".css" : "text/css", + ".html" : "text/html" } if ext in extension_map: return extension_map[ext] @@ -444,6 +449,7 @@ class _Handler (WgetHTTPRequestHandler): class HTTPd (threading.Thread): server_class = StoppableHTTPServer handler = _Handler + def __init__ (self, addr=None): threading.Thread.__init__ (self) if addr is None: @@ -452,16 +458,14 @@ class HTTPd (threading.Thread): self.server_address = self.server_inst.socket.getsockname()[:2] def run (self): - self.server_inst.serve_forever () + self.server_inst.serve_forever () def server_conf (self, file_list, server_rules): self.server_inst.server_conf (file_list, server_rules) - def server_sett (self, settings): - self.server_inst.server_sett (settings) class HTTPSd (HTTPd): - server_class = HTTPSServer + server_class = HTTPSServer # vim: set ts=4 sts=4 sw=4 tw=80 et : diff --git a/testenv/test/base_test.py b/testenv/test/base_test.py index 369370a3..8bf8ea5f 100644 --- a/testenv/test/base_test.py +++ b/testenv/test/base_test.py @@ -28,9 +28,9 @@ class BaseTest: Attributes should not be defined outside __init__. """ self.name = name - self.pre_configs = pre_hook or {} # if pre_hook == None, then - # {} (an empty dict object) is - # passed to self.pre_configs + self.pre_configs = pre_hook or {} # if pre_hook == None, then + # {} (an empty dict object) is + # passed to self.pre_configs self.test_params = test_params or {} self.post_configs = post_hook or {} self.protocols = protocols @@ -98,12 +98,13 @@ class BaseTest: test_path = os.path.abspath(".") wget_path = os.path.abspath(os.path.join(test_path, "..", '..', 'src', "wget")) + wget_options = '--debug --no-config %s' % self.wget_options if os.getenv("VALGRIND_TESTS"): valgrind_test = "valgrind --error-exitcode=301 --leak-check=full" else: valgrind_test = "" - cmd_line = '%s %s %s ' % (valgrind_test, wget_path, self.wget_options) + cmd_line = '%s %s %s ' % (valgrind_test, wget_path, wget_options) for protocol, urls, domain in zip(self.protocols, self.urls, self.domains): diff --git a/testenv/test/http_test.py b/testenv/test/http_test.py index fe2254de..230eff8e 100644 --- a/testenv/test/http_test.py +++ b/testenv/test/http_test.py @@ -7,9 +7,10 @@ class HTTPTest(BaseTest): """ Class for HTTP Tests. """ - # Temp Notes: It is expected that when pre-hook functions are executed, only an empty test-dir exists. - # pre-hook functions are executed just prior to the call to Wget is made. - # post-hook functions will be executed immediately after the call to Wget returns. + # Temp Notes: It is expected that when pre-hook functions are executed, + # only an empty test-dir exists. pre-hook functions are executed just prior + # to the call to Wget is made. post-hook functions will be executed + # immediately after the call to Wget returns. def __init__(self, name="Unnamed Test", |