Merge branch 'develop' of https://github.com/hyperledger/iroha into f…

…ix/itf-fake-peer-on-demand-ordering

Signed-off-by: Mikhail Boldyrev <[email protected]>

CMakeLists.txt

@@ -60,7 +60,7 @@ if(COVERAGE)
   add_custom_target(coverage.info
       COMMAND ${LCOV_PROGRAM} --config-file ${LCOV_CONFIG_FILE} -o ${PROJECT_BINARY_DIR}/reports/coverage.info -c -d ${PROJECT_BINARY_DIR}
       COMMAND ${LCOV_PROGRAM} --config-file ${LCOV_CONFIG_FILE} -o ${PROJECT_BINARY_DIR}/reports/coverage.info -a ${PROJECT_BINARY_DIR}/reports/coverage.init.info -a ${PROJECT_BINARY_DIR}/reports/coverage.info
-      COMMAND ${LCOV_PROGRAM} --config-file ${LCOV_CONFIG_FILE} -o ${PROJECT_BINARY_DIR}/reports/coverage.info -r ${PROJECT_BINARY_DIR}/reports/coverage.info '/usr*' '${CMAKE_SOURCE_DIR}/external/*' '${CMAKE_SOURCE_DIR}/schema/*'
+      COMMAND ${LCOV_PROGRAM} --config-file ${LCOV_CONFIG_FILE} -o ${PROJECT_BINARY_DIR}/reports/coverage.info -r ${PROJECT_BINARY_DIR}/reports/coverage.info '/usr*' '/Library/Developer/CommandLineTools/*' '${PROJECT_BINARY_DIR}/*' '${CMAKE_BINARY_DIR}/*' '${CMAKE_SOURCE_DIR}/test/*'
       )
   set(REPORT_DIR ${CMAKE_BINARY_DIR}/reports)
   file(MAKE_DIRECTORY ${REPORT_DIR})

CONTRIBUTING.md

@@ -7,7 +7,8 @@ The following is a short set of guidelines for contributing to Iroha.
 
 ##### [How Can I Contribute?](#how-can-i-contribute-1)
 
-- [Reporting bugs](#reporting-bugs)
+- [Reporting Bugs](#reporting-bugs)
+- [Reporting Vulnerabilities](#reporting-vulnerabilities)
 - [Suggesting Improvements](#suggesting-improvements)
 - [Asking Questions](#asking-questions)
 - [Your First Code Contribution](#your-first-code-contribution)
@@ -24,7 +25,7 @@ The following is a short set of guidelines for contributing to Iroha.
 
 ## How Can I Contribute?
 
-### Reporting bugs
+### Reporting Bugs
 
 *Bug* is an error, design flaw, failure or fault in Iroha that causes it to produce an incorrect or unexpected result, or to behave in unintended ways.
 
@@ -42,6 +43,17 @@ To submit a bug, [create new issue](https://jira.hyperledger.org/secure/CreateIs
 | Environment             | Your OS, device's specs, Virtual Environment if you use one, version of Iroha etc. |
 
 
+### Reporting Vulnerabilities
+
+While we try to be proactive in preventing security problems, we do not assume they’ll never come up.
+
+It is standard practice to responsibly and privately disclose to the vendor (Hyperledger organization)
+a security problem before publicizing, so a fix can be prepared, and damage from the vulnerability minimized.
+
+Before the First Major Release (1.0) all vulnerabilities are considered to be bugs, so feel free to submit them as described above.
+After the First Major Release please utilize [a bug bounty program here](https://hackerone.com/hyperledger) in order to submit vulnerabilities and get your reward.
+
+In any case — feel free to reach to any of existing maintainers in Rocket.Chat private messages or in an e-mail (check CONTRIBUTORS.md file) if you want to discuss whether your discovery is a vulnerability or a bug.
 
 ### Suggesting Improvements
 

deploy/ansible/playbooks/iroha-docker/main.yml

@@ -5,4 +5,4 @@
     - { role: docker, tags: docker }
     # - { role: iroha-docker, tags: iroha-docker }
   vars:
-    hostnames: []
+    hostnames: []

deploy/ansible/roles/iroha-docker/defaults/main.yml

@@ -47,6 +47,9 @@ postgres_docker_tag: '9.5'
 iroha_peer_port: 10001
 # As above but for base Torii port
 iroha_torii_port: 50051
+# Enables 2 phase commit optimization. Postgres documentation suggests setting it to a max number of
+# connections, which is 100 by default
+iroha_postgres_max_prepared_transactions: 100
 
 # Rest of the options affect Iroha configuration
 # See https://iroha.readthedocs.io/en/latest/guides/configuration.html#configuration
@@ -60,3 +63,6 @@ iroha_postgres_password: postgres
 iroha_postgres_host: iroha-postgres
 iroha_postgres_port: 5432
 iroha_blockstore_path: /tmp/block_store
+#iroha_mst_expiration_time: 1440
+#iroha_max_rounds_delay: 3000
+#iroha_stale_stream_max_rounds: 2

deploy/ansible/roles/iroha-docker/files/config_gen.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+set -e
 
 FORCE_OVERWRITE=0
 OUT_DIR='config'

deploy/ansible/roles/iroha-docker/tasks/deploy.yml

@@ -59,8 +59,13 @@
     name: "{{ iroha_network_name }}"
   when: not overlay_network
 
+- name: pull Iroha
+  command: docker-compose pull
+  args:
+    chdir: "{{ deploy_dir }}"
+  changed_when: False
+
 - name: run Iroha
   docker_service:
     project_src: "{{ deploy_dir }}"
     project_name: ansible_iroha
-    pull: yes

deploy/ansible/roles/iroha-docker/templates/config.docker.j2

@@ -7,4 +7,13 @@
   "proposal_delay" : {{ iroha_proposal_delay }},
   "vote_delay" : {{ iroha_vote_delay }},
   "mst_enable" : {{ iroha_mst_enable }}
+  {% if iroha_mst_expiration_time is defined %}
+    ,"mst_expiration_time": {{ iroha_mst_expiration_time }}
+  {% endif %}
+  {% if iroha_max_rounds_delay is defined %}
+    ,"max_rounds_delay": {{ iroha_max_rounds_delay }}
+  {% endif %}
+  {% if iroha_stale_stream_max_rounds is defined %}
+    ,"stale_stream_max_rounds": {{ iroha_stale_stream_max_rounds }}
+  {% endif %}
 }

deploy/ansible/roles/iroha-docker/templates/docker-compose.yml.j2

@@ -39,6 +39,9 @@ services:
       - psql_storage-{{ node.human_hostname }}:/var/lib/postgresql/data
     networks:
       - iroha-db-net
+{% if iroha_postgres_max_prepared_transactions is defined %}
+    command: -c max_prepared_transactions={{ iroha_postgres_max_prepared_transactions }}
+{% endif %}
 
 {% endfor %}
 

docs/source/guides/configuration.rst

@@ -15,7 +15,10 @@ at ``example/config.sample``
     "max_proposal_size": 10,
     "proposal_delay": 5000,
     "vote_delay": 5000,
-    "mst_enable" : false
+    "mst_enable" : false,
+    "mst_expiration_time" : 1440,
+    "max_rounds_delay": 3000,
+    "stale_stream_max_rounds": 2
   }
 
 As you can see, configuration file is a valid ``json`` structure. Let's go 
@@ -53,9 +56,31 @@ Environment-specific parameters
   next peer. Optimal value depends heavily on the amount of Iroha peers in the
   network (higher amount of nodes requires longer ``vote_delay``). We recommend
   to start with 100-1000 milliseconds.
-- ``mst_enable`` enables or disables multisignature transaction support in
-  Iroha. We recommend setting this parameter to ``false`` at the moment until
-  you really need it.
+- ``mst_enable`` enables or disables multisignature transaction network
+  transport in Iroha. We recommend setting this parameter to ``false`` at the
+  moment until you really need it.
+  ``mst_expiration_time`` is an optional parameter specifying the time period
+  in which a not fully signed transaction (or a batch) is considered expired
+  (in minutes).
+  The default value is 1440.
+- ``max_rounds_delay`` is an optional parameter specifying the maximum delay
+  between two consensus rounds (in milliseconds).
+  The default value is 3000.
+  When Iroha is idle, it gradually increases the delay to reduce CPU, network
+  and logging load.
+  However too long delay may be unwanted when first transactions arrive after a
+  long idle time.
+  This parameter allows users to find an optimal value in a tradeoff between
+  resource consumption and the delay of getting back to work after an idle
+  period.
+- ``stale_stream_max_rounds`` is an optional parameter specifying the maximum
+  amount of rounds to keep an open status stream while no status update is
+  reported.
+  The default value is 2.
+  Increasing this value reduces the amount of times a client must reconnect to
+  track a transaction if for some reason it is not updated with new rounds.
+  However large values increase the average number of connected clients during
+  each round.
 
 Logging
 -------

irohad/ametsuchi/CMakeLists.txt

@@ -20,6 +20,8 @@ add_library(ametsuchi
     impl/tx_presence_cache_impl.cpp
     impl/in_memory_block_storage.cpp
     impl/in_memory_block_storage_factory.cpp
+    impl/flat_file_block_storage.cpp
+    impl/flat_file_block_storage_factory.cpp
     )
 
 target_link_libraries(ametsuchi

irohad/ametsuchi/block_storage.hpp

@@ -28,9 +28,6 @@ namespace iroha {
       virtual bool insert(
           std::shared_ptr<const shared_model::interface::Block> block) = 0;
 
-      [[deprecated("Use shared_ptr")]] virtual bool insert(
-          const shared_model::interface::Block &block) = 0;
-
       /**
        * Get block with given height
        * @return block if exists, boost::none otherwise

irohad/ametsuchi/block_storage_factory.hpp

@@ -12,7 +12,6 @@
 
 namespace iroha {
   namespace ametsuchi {
-
     /**
      * Creates a block storage
      */

irohad/ametsuchi/impl/flat_file/flat_file.cpp

@@ -18,6 +18,7 @@
 
 using namespace iroha::ametsuchi;
 using Identifier = FlatFile::Identifier;
+using BlockIdCollectionType = FlatFile::BlockIdCollectionType;
 
 // ----------| public API |----------
 
@@ -27,29 +28,45 @@ std::string FlatFile::id_to_name(Identifier id) {
   return os.str();
 }
 
+boost::optional<Identifier> FlatFile::name_to_id(const std::string &name) {
+  if (name.size() != FlatFile::DIGIT_CAPACITY) {
+    return boost::none;
+  }
+  try {
+    auto id = std::stoul(name);
+    return boost::make_optional<Identifier>(id);
+  } catch (const std::exception &e) {
+    return boost::none;
+  }
+}
+
 boost::optional<std::unique_ptr<FlatFile>> FlatFile::create(
-    const std::string &path,
-    logger::LoggerPtr log) {
+    const std::string &path, logger::LoggerPtr log) {
   boost::system::error_code err;
   if (not boost::filesystem::is_directory(path, err)
       and not boost::filesystem::create_directory(path, err)) {
     log->error("Cannot create storage dir: {}\n{}", path, err.message());
     return boost::none;
   }
 
-  auto res = FlatFile::check_consistency(path, log);
-  return std::make_unique<FlatFile>(*res, path, private_tag{}, std::move(log));
+  BlockIdCollectionType files_found;
+  for (auto it = boost::filesystem::directory_iterator{path};
+       it != boost::filesystem::directory_iterator{};
+       ++it) {
+    if (auto id = FlatFile::name_to_id(it->path().filename().string())) {
+      files_found.insert(*id);
+    } else {
+      boost::filesystem::remove(it->path());
+    }
+  }
+
+  return std::make_unique<FlatFile>(
+      path, std::move(files_found), private_tag{}, std::move(log));
 }
 
 bool FlatFile::add(Identifier id, const Bytes &block) {
   // TODO(x3medima17): Change bool to generic Result return type
 
-  if (id != current_id_ + 1) {
-    log_->warn("Cannot append non-consecutive block");
-    return false;
-  }
-
-  auto next_id = id;
   const auto file_name = boost::filesystem::path{dump_dir_} / id_to_name(id);
 
   // Write block to binary file
@@ -71,8 +88,7 @@ bool FlatFile::add(Identifier id, const Bytes &block) {
   file.write(reinterpret_cast<const char *>(block.data()),
              block.size() * val_size);
 
-  // Update internals, release lock
-  current_id_ = next_id;
+  available_blocks_.insert(id);
   return true;
 }
 
@@ -100,50 +116,24 @@ std::string FlatFile::directory() const {
 }
 
 Identifier FlatFile::last_id() const {
-  return current_id_.load();
+  return (available_blocks_.empty()) ? 0 : *available_blocks_.rbegin();
 }
 
 void FlatFile::dropAll() {
   iroha::remove_dir_contents(dump_dir_, log_);
-  auto res = FlatFile::check_consistency(dump_dir_, log_);
-  current_id_.store(*res);
+  available_blocks_.clear();
+}
+
+const BlockIdCollectionType &FlatFile::blockIdentifiers() const {
+  return available_blocks_;
 }
 
 // ----------| private API |----------
 
-FlatFile::FlatFile(Identifier current_id,
-                   const std::string &path,
+FlatFile::FlatFile(std::string path,
+                   BlockIdCollectionType existing_files,
                    FlatFile::private_tag,
                    logger::LoggerPtr log)
-    : dump_dir_(path), log_{std::move(log)} {
-  current_id_.store(current_id);
-}
-
-boost::optional<Identifier> FlatFile::check_consistency(
-    const std::string &dump_dir, logger::LoggerPtr log) {
-  if (dump_dir.empty()) {
-    log->error("check_consistency({}), not directory", dump_dir);
-    return boost::none;
-  }
-
-  auto const files = [&dump_dir] {
-    std::vector<boost::filesystem::path> ps;
-    std::copy(boost::filesystem::directory_iterator{dump_dir},
-              boost::filesystem::directory_iterator{},
-              std::back_inserter(ps));
-    std::sort(ps.begin(), ps.end(), std::less<boost::filesystem::path>());
-    return ps;
-  }();
-
-  auto const missing = boost::range::find_if(
-      files | boost::adaptors::indexed(1), [](const auto &it) {
-        return FlatFile::id_to_name(it.index()) != it.value().filename();
-      });
-
-  std::for_each(
-      missing.get(), files.cend(), [](const boost::filesystem::path &p) {
-        boost::filesystem::remove(p);
-      });
-
-  return missing.get() - files.cbegin();
-}
+    : dump_dir_(std::move(path)),
+      available_blocks_(std::move(existing_files)),
+      log_{std::move(log)} {}